arch/s390/kvm/kvm-s390.c

   1 /*
   2  * hosting zSeries kernel virtual machines
   3  *
   4  * Copyright IBM Corp. 2008, 2009
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License (version 2 only)
   8  * as published by the Free Software Foundation.
   9  *
  10  *    Author(s): Carsten Otte <cotte@de.ibm.com>
  11  *               Christian Borntraeger <borntraeger@de.ibm.com>
  12  *               Heiko Carstens <heiko.carstens@de.ibm.com>
  13  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
  14  *               Jason J. Herne <jjherne@us.ibm.com>
  15  */
  16
  17 #include <linux/compiler.h>
  18 #include <linux/err.h>
  19 #include <linux/fs.h>
  20 #include <linux/hrtimer.h>
  21 #include <linux/init.h>
  22 #include <linux/kvm.h>
  23 #include <linux/kvm_host.h>
  24 #include <linux/mman.h>
  25 #include <linux/module.h>
  26 #include <linux/moduleparam.h>
  27 #include <linux/random.h>
  28 #include <linux/slab.h>
  29 #include <linux/timer.h>
  30 #include <linux/vmalloc.h>
  31 #include <linux/bitmap.h>
  32 #include <asm/asm-offsets.h>
  33 #include <asm/lowcore.h>
  34 #include <asm/stp.h>
  35 #include <asm/pgtable.h>
  36 #include <asm/gmap.h>
  37 #include <asm/nmi.h>
  38 #include <asm/switch_to.h>
  39 #include <asm/isc.h>
  40 #include <asm/sclp.h>
  41 #include <asm/cpacf.h>
  42 #include <asm/timex.h>
  43 #include "kvm-s390.h"
  44 #include "gaccess.h"
  45
  46 #define KMSG_COMPONENT "kvm-s390"
  47 #undef pr_fmt
  48 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  49
  50 #define CREATE_TRACE_POINTS
  51 #include "trace.h"
  52 #include "trace-s390.h"
  53
  54 #define MEM_OP_MAX_SIZE 65536   /* Maximum transfer size for KVM_S390_MEM_OP */
  55 #define LOCAL_IRQS 32
  56 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
  57                            (KVM_MAX_VCPUS + LOCAL_IRQS))
  58
  59 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  60
  61 struct kvm_stats_debugfs_item debugfs_entries[] = {
  62         { "userspace_handled", VCPU_STAT(exit_userspace) },
  63         { "exit_null", VCPU_STAT(exit_null) },
  64         { "exit_validity", VCPU_STAT(exit_validity) },
  65         { "exit_stop_request", VCPU_STAT(exit_stop_request) },
  66         { "exit_external_request", VCPU_STAT(exit_external_request) },
  67         { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
  68         { "exit_instruction", VCPU_STAT(exit_instruction) },
  69         { "exit_pei", VCPU_STAT(exit_pei) },
  70         { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
  71         { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
  72         { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
  73         { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
  74         { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
  75         { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
  76         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  77         { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
  78         { "instruction_lctl", VCPU_STAT(instruction_lctl) },
  79         { "instruction_stctl", VCPU_STAT(instruction_stctl) },
  80         { "instruction_stctg", VCPU_STAT(instruction_stctg) },
  81         { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
  82         { "deliver_external_call", VCPU_STAT(deliver_external_call) },
  83         { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
  84         { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
  85         { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
  86         { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
  87         { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
  88         { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
  89         { "exit_wait_state", VCPU_STAT(exit_wait_state) },
  90         { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
  91         { "instruction_stidp", VCPU_STAT(instruction_stidp) },
  92         { "instruction_spx", VCPU_STAT(instruction_spx) },
  93         { "instruction_stpx", VCPU_STAT(instruction_stpx) },
  94         { "instruction_stap", VCPU_STAT(instruction_stap) },
  95         { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
  96         { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
  97         { "instruction_stsch", VCPU_STAT(instruction_stsch) },
  98         { "instruction_chsc", VCPU_STAT(instruction_chsc) },
  99         { "instruction_essa", VCPU_STAT(instruction_essa) },
 100         { "instruction_stsi", VCPU_STAT(instruction_stsi) },
 101         { "instruction_stfl", VCPU_STAT(instruction_stfl) },
 102         { "instruction_tprot", VCPU_STAT(instruction_tprot) },
 103         { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
 104         { "instruction_sie", VCPU_STAT(instruction_sie) },
 105         { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
 106         { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
 107         { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
 108         { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
 109         { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
 110         { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
 111         { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
 112         { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
 113         { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
 114         { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
 115         { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
 116         { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
 117         { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
 118         { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
 119         { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
 120         { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
 121         { "diagnose_10", VCPU_STAT(diagnose_10) },
 122         { "diagnose_44", VCPU_STAT(diagnose_44) },
 123         { "diagnose_9c", VCPU_STAT(diagnose_9c) },
 124         { "diagnose_258", VCPU_STAT(diagnose_258) },
 125         { "diagnose_308", VCPU_STAT(diagnose_308) },
 126         { "diagnose_500", VCPU_STAT(diagnose_500) },
 127         { NULL }
 128 };
 129
 130 /* allow nested virtualization in KVM (if enabled by user space) */
 131 static int nested;
 132 module_param(nested, int, S_IRUGO);
 133 MODULE_PARM_DESC(nested, "Nested virtualization support");
 134
 135 /* upper facilities limit for kvm */
 136 unsigned long kvm_s390_fac_list_mask[16] = { FACILITIES_KVM };
 137
 138 unsigned long kvm_s390_fac_list_mask_size(void)
 139 {
 140         BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
 141         return ARRAY_SIZE(kvm_s390_fac_list_mask);
 142 }
 143
 144 /* available cpu features supported by kvm */
 145 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
 146 /* available subfunctions indicated via query / "test bit" */
 147 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
 148
 149 static struct gmap_notifier gmap_notifier;
 150 static struct gmap_notifier vsie_gmap_notifier;
 151 debug_info_t *kvm_s390_dbf;
 152
 153 /* Section: not file related */
 154 int kvm_arch_hardware_enable(void)
 155 {
 156         /* every s390 is virtualization enabled ;-) */
 157         return 0;
 158 }
 159
 160 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
 161                               unsigned long end);
 162
 163 /*
 164  * This callback is executed during stop_machine(). All CPUs are therefore
 165  * temporarily stopped. In order not to change guest behavior, we have to
 166  * disable preemption whenever we touch the epoch of kvm and the VCPUs,
 167  * so a CPU won't be stopped while calculating with the epoch.
 168  */
 169 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
 170                           void *v)
 171 {
 172         struct kvm *kvm;
 173         struct kvm_vcpu *vcpu;
 174         int i;
 175         unsigned long long *delta = v;
 176
 177         list_for_each_entry(kvm, &vm_list, vm_list) {
 178                 kvm->arch.epoch -= *delta;
 179                 kvm_for_each_vcpu(i, vcpu, kvm) {
 180                         vcpu->arch.sie_block->epoch -= *delta;
 181                         if (vcpu->arch.cputm_enabled)
 182                                 vcpu->arch.cputm_start += *delta;
 183                         if (vcpu->arch.vsie_block)
 184                                 vcpu->arch.vsie_block->epoch -= *delta;
 185                 }
 186         }
 187         return NOTIFY_OK;
 188 }
 189
 190 static struct notifier_block kvm_clock_notifier = {
 191         .notifier_call = kvm_clock_sync,
 192 };
 193
 194 int kvm_arch_hardware_setup(void)
 195 {
 196         gmap_notifier.notifier_call = kvm_gmap_notifier;
 197         gmap_register_pte_notifier(&gmap_notifier);
 198         vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
 199         gmap_register_pte_notifier(&vsie_gmap_notifier);
 200         atomic_notifier_chain_register(&s390_epoch_delta_notifier,
 201                                        &kvm_clock_notifier);
 202         return 0;
 203 }
 204
 205 void kvm_arch_hardware_unsetup(void)
 206 {
 207         gmap_unregister_pte_notifier(&gmap_notifier);
 208         gmap_unregister_pte_notifier(&vsie_gmap_notifier);
 209         atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
 210                                          &kvm_clock_notifier);
 211 }
 212
 213 static void allow_cpu_feat(unsigned long nr)
 214 {
 215         set_bit_inv(nr, kvm_s390_available_cpu_feat);
 216 }
 217
 218 static inline int plo_test_bit(unsigned char nr)
 219 {
 220         register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
 221         int cc;
 222
 223         asm volatile(
 224                 /* Parameter registers are ignored for "test bit" */
 225                 "       plo     0,0,0,0(0)\n"
 226                 "       ipm     %0\n"
 227                 "       srl     %0,28\n"
 228                 : "=d" (cc)
 229                 : "d" (r0)
 230                 : "cc");
 231         return cc == 0;
 232 }
 233
 234 static void kvm_s390_cpu_feat_init(void)
 235 {
 236         int i;
 237
 238         for (i = 0; i < 256; ++i) {
 239                 if (plo_test_bit(i))
 240                         kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
 241         }
 242
 243         if (test_facility(28)) /* TOD-clock steering */
 244                 ptff(kvm_s390_available_subfunc.ptff,
 245                      sizeof(kvm_s390_available_subfunc.ptff),
 246                      PTFF_QAF);
 247
 248         if (test_facility(17)) { /* MSA */
 249                 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
 250                               kvm_s390_available_subfunc.kmac);
 251                 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
 252                               kvm_s390_available_subfunc.kmc);
 253                 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
 254                               kvm_s390_available_subfunc.km);
 255                 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
 256                               kvm_s390_available_subfunc.kimd);
 257                 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
 258                               kvm_s390_available_subfunc.klmd);
 259         }
 260         if (test_facility(76)) /* MSA3 */
 261                 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
 262                               kvm_s390_available_subfunc.pckmo);
 263         if (test_facility(77)) { /* MSA4 */
 264                 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
 265                               kvm_s390_available_subfunc.kmctr);
 266                 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
 267                               kvm_s390_available_subfunc.kmf);
 268                 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
 269                               kvm_s390_available_subfunc.kmo);
 270                 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
 271                               kvm_s390_available_subfunc.pcc);
 272         }
 273         if (test_facility(57)) /* MSA5 */
 274                 __cpacf_query(CPACF_PPNO, (cpacf_mask_t *)
 275                               kvm_s390_available_subfunc.ppno);
 276
 277         if (MACHINE_HAS_ESOP)
 278                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
 279         /*
 280          * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
 281          * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
 282          */
 283         if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
 284             !test_facility(3) || !nested)
 285                 return;
 286         allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
 287         if (sclp.has_64bscao)
 288                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
 289         if (sclp.has_siif)
 290                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
 291         if (sclp.has_gpere)
 292                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
 293         if (sclp.has_gsls)
 294                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
 295         if (sclp.has_ib)
 296                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
 297         if (sclp.has_cei)
 298                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
 299         if (sclp.has_ibs)
 300                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
 301         /*
 302          * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
 303          * all skey handling functions read/set the skey from the PGSTE
 304          * instead of the real storage key.
 305          *
 306          * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
 307          * pages being detected as preserved although they are resident.
 308          *
 309          * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
 310          * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
 311          *
 312          * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
 313          * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
 314          * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
 315          *
 316          * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
 317          * cannot easily shadow the SCA because of the ipte lock.
 318          */
 319 }
 320
 321 int kvm_arch_init(void *opaque)
 322 {
 323         kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
 324         if (!kvm_s390_dbf)
 325                 return -ENOMEM;
 326
 327         if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
 328                 debug_unregister(kvm_s390_dbf);
 329                 return -ENOMEM;
 330         }
 331
 332         kvm_s390_cpu_feat_init();
 333
 334         /* Register floating interrupt controller interface. */
 335         return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
 336 }
 337
 338 void kvm_arch_exit(void)
 339 {
 340         debug_unregister(kvm_s390_dbf);
 341 }
 342
 343 /* Section: device related */
 344 long kvm_arch_dev_ioctl(struct file *filp,
 345                         unsigned int ioctl, unsigned long arg)
 346 {
 347         if (ioctl == KVM_S390_ENABLE_SIE)
 348                 return s390_enable_sie();
 349         return -EINVAL;
 350 }
 351
 352 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 353 {
 354         int r;
 355
 356         switch (ext) {
 357         case KVM_CAP_S390_PSW:
 358         case KVM_CAP_S390_GMAP:
 359         case KVM_CAP_SYNC_MMU:
 360 #ifdef CONFIG_KVM_S390_UCONTROL
 361         case KVM_CAP_S390_UCONTROL:
 362 #endif
 363         case KVM_CAP_ASYNC_PF:
 364         case KVM_CAP_SYNC_REGS:
 365         case KVM_CAP_ONE_REG:
 366         case KVM_CAP_ENABLE_CAP:
 367         case KVM_CAP_S390_CSS_SUPPORT:
 368         case KVM_CAP_IOEVENTFD:
 369         case KVM_CAP_DEVICE_CTRL:
 370         case KVM_CAP_ENABLE_CAP_VM:
 371         case KVM_CAP_S390_IRQCHIP:
 372         case KVM_CAP_VM_ATTRIBUTES:
 373         case KVM_CAP_MP_STATE:
 374         case KVM_CAP_IMMEDIATE_EXIT:
 375         case KVM_CAP_S390_INJECT_IRQ:
 376         case KVM_CAP_S390_USER_SIGP:
 377         case KVM_CAP_S390_USER_STSI:
 378         case KVM_CAP_S390_SKEYS:
 379         case KVM_CAP_S390_IRQ_STATE:
 380         case KVM_CAP_S390_USER_INSTR0:
 381                 r = 1;
 382                 break;
 383         case KVM_CAP_S390_MEM_OP:
 384                 r = MEM_OP_MAX_SIZE;
 385                 break;
 386         case KVM_CAP_NR_VCPUS:
 387         case KVM_CAP_MAX_VCPUS:
 388                 r = KVM_S390_BSCA_CPU_SLOTS;
 389                 if (!kvm_s390_use_sca_entries())
 390                         r = KVM_MAX_VCPUS;
 391                 else if (sclp.has_esca && sclp.has_64bscao)
 392                         r = KVM_S390_ESCA_CPU_SLOTS;
 393                 break;
 394         case KVM_CAP_NR_MEMSLOTS:
 395                 r = KVM_USER_MEM_SLOTS;
 396                 break;
 397         case KVM_CAP_S390_COW:
 398                 r = MACHINE_HAS_ESOP;
 399                 break;
 400         case KVM_CAP_S390_VECTOR_REGISTERS:
 401                 r = MACHINE_HAS_VX;
 402                 break;
 403         case KVM_CAP_S390_RI:
 404                 r = test_facility(64);
 405                 break;
 406         default:
 407                 r = 0;
 408         }
 409         return r;
 410 }
 411
 412 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
 413                                         struct kvm_memory_slot *memslot)
 414 {
 415         gfn_t cur_gfn, last_gfn;
 416         unsigned long address;
 417         struct gmap *gmap = kvm->arch.gmap;
 418
 419         /* Loop over all guest pages */
 420         last_gfn = memslot->base_gfn + memslot->npages;
 421         for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
 422                 address = gfn_to_hva_memslot(memslot, cur_gfn);
 423
 424                 if (test_and_clear_guest_dirty(gmap->mm, address))
 425                         mark_page_dirty(kvm, cur_gfn);
 426                 if (fatal_signal_pending(current))
 427                         return;
 428                 cond_resched();
 429         }
 430 }
 431
 432 /* Section: vm related */
 433 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
 434
 435 /*
 436  * Get (and clear) the dirty memory log for a memory slot.
 437  */
 438 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 439                                struct kvm_dirty_log *log)
 440 {
 441         int r;
 442         unsigned long n;
 443         struct kvm_memslots *slots;
 444         struct kvm_memory_slot *memslot;
 445         int is_dirty = 0;
 446
 447         if (kvm_is_ucontrol(kvm))
 448                 return -EINVAL;
 449
 450         mutex_lock(&kvm->slots_lock);
 451
 452         r = -EINVAL;
 453         if (log->slot >= KVM_USER_MEM_SLOTS)
 454                 goto out;
 455
 456         slots = kvm_memslots(kvm);
 457         memslot = id_to_memslot(slots, log->slot);
 458         r = -ENOENT;
 459         if (!memslot->dirty_bitmap)
 460                 goto out;
 461
 462         kvm_s390_sync_dirty_log(kvm, memslot);
 463         r = kvm_get_dirty_log(kvm, log, &is_dirty);
 464         if (r)
 465                 goto out;
 466
 467         /* Clear the dirty log */
 468         if (is_dirty) {
 469                 n = kvm_dirty_bitmap_bytes(memslot);
 470                 memset(memslot->dirty_bitmap, 0, n);
 471         }
 472         r = 0;
 473 out:
 474         mutex_unlock(&kvm->slots_lock);
 475         return r;
 476 }
 477
 478 static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
 479 {
 480         unsigned int i;
 481         struct kvm_vcpu *vcpu;
 482
 483         kvm_for_each_vcpu(i, vcpu, kvm) {
 484                 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
 485         }
 486 }
 487
 488 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
 489 {
 490         int r;
 491
 492         if (cap->flags)
 493                 return -EINVAL;
 494
 495         switch (cap->cap) {
 496         case KVM_CAP_S390_IRQCHIP:
 497                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
 498                 kvm->arch.use_irqchip = 1;
 499                 r = 0;
 500                 break;
 501         case KVM_CAP_S390_USER_SIGP:
 502                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
 503                 kvm->arch.user_sigp = 1;
 504                 r = 0;
 505                 break;
 506         case KVM_CAP_S390_VECTOR_REGISTERS:
 507                 mutex_lock(&kvm->lock);
 508                 if (kvm->created_vcpus) {
 509                         r = -EBUSY;
 510                 } else if (MACHINE_HAS_VX) {
 511                         set_kvm_facility(kvm->arch.model.fac_mask, 129);
 512                         set_kvm_facility(kvm->arch.model.fac_list, 129);
 513                         if (test_facility(134)) {
 514                                 set_kvm_facility(kvm->arch.model.fac_mask, 134);
 515                                 set_kvm_facility(kvm->arch.model.fac_list, 134);
 516                         }
 517                         if (test_facility(135)) {
 518                                 set_kvm_facility(kvm->arch.model.fac_mask, 135);
 519                                 set_kvm_facility(kvm->arch.model.fac_list, 135);
 520                         }
 521                         r = 0;
 522                 } else
 523                         r = -EINVAL;
 524                 mutex_unlock(&kvm->lock);
 525                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
 526                          r ? "(not available)" : "(success)");
 527                 break;
 528         case KVM_CAP_S390_RI:
 529                 r = -EINVAL;
 530                 mutex_lock(&kvm->lock);
 531                 if (kvm->created_vcpus) {
 532                         r = -EBUSY;
 533                 } else if (test_facility(64)) {
 534                         set_kvm_facility(kvm->arch.model.fac_mask, 64);
 535                         set_kvm_facility(kvm->arch.model.fac_list, 64);
 536                         r = 0;
 537                 }
 538                 mutex_unlock(&kvm->lock);
 539                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
 540                          r ? "(not available)" : "(success)");
 541                 break;
 542         case KVM_CAP_S390_USER_STSI:
 543                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
 544                 kvm->arch.user_stsi = 1;
 545                 r = 0;
 546                 break;
 547         case KVM_CAP_S390_USER_INSTR0:
 548                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
 549                 kvm->arch.user_instr0 = 1;
 550                 icpt_operexc_on_all_vcpus(kvm);
 551                 r = 0;
 552                 break;
 553         default:
 554                 r = -EINVAL;
 555                 break;
 556         }
 557         return r;
 558 }
 559
 560 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 561 {
 562         int ret;
 563
 564         switch (attr->attr) {
 565         case KVM_S390_VM_MEM_LIMIT_SIZE:
 566                 ret = 0;
 567                 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
 568                          kvm->arch.mem_limit);
 569                 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
 570                         ret = -EFAULT;
 571                 break;
 572         default:
 573                 ret = -ENXIO;
 574                 break;
 575         }
 576         return ret;
 577 }
 578
 579 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 580 {
 581         int ret;
 582         unsigned int idx;
 583         switch (attr->attr) {
 584         case KVM_S390_VM_MEM_ENABLE_CMMA:
 585                 ret = -ENXIO;
 586                 if (!sclp.has_cmma)
 587                         break;
 588
 589                 ret = -EBUSY;
 590                 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
 591                 mutex_lock(&kvm->lock);
 592                 if (!kvm->created_vcpus) {
 593                         kvm->arch.use_cmma = 1;
 594                         ret = 0;
 595                 }
 596                 mutex_unlock(&kvm->lock);
 597                 break;
 598         case KVM_S390_VM_MEM_CLR_CMMA:
 599                 ret = -ENXIO;
 600                 if (!sclp.has_cmma)
 601                         break;
 602                 ret = -EINVAL;
 603                 if (!kvm->arch.use_cmma)
 604                         break;
 605
 606                 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
 607                 mutex_lock(&kvm->lock);
 608                 idx = srcu_read_lock(&kvm->srcu);
 609                 s390_reset_cmma(kvm->arch.gmap->mm);
 610                 srcu_read_unlock(&kvm->srcu, idx);
 611                 mutex_unlock(&kvm->lock);
 612                 ret = 0;
 613                 break;
 614         case KVM_S390_VM_MEM_LIMIT_SIZE: {
 615                 unsigned long new_limit;
 616
 617                 if (kvm_is_ucontrol(kvm))
 618                         return -EINVAL;
 619
 620                 if (get_user(new_limit, (u64 __user *)attr->addr))
 621                         return -EFAULT;
 622
 623                 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
 624                     new_limit > kvm->arch.mem_limit)
 625                         return -E2BIG;
 626
 627                 if (!new_limit)
 628                         return -EINVAL;
 629
 630                 /* gmap_create takes last usable address */
 631                 if (new_limit != KVM_S390_NO_MEM_LIMIT)
 632                         new_limit -= 1;
 633
 634                 ret = -EBUSY;
 635                 mutex_lock(&kvm->lock);
 636                 if (!kvm->created_vcpus) {
 637                         /* gmap_create will round the limit up */
 638                         struct gmap *new = gmap_create(current->mm, new_limit);
 639
 640                         if (!new) {
 641                                 ret = -ENOMEM;
 642                         } else {
 643                                 gmap_remove(kvm->arch.gmap);
 644                                 new->private = kvm;
 645                                 kvm->arch.gmap = new;
 646                                 ret = 0;
 647                         }
 648                 }
 649                 mutex_unlock(&kvm->lock);
 650                 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
 651                 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
 652                          (void *) kvm->arch.gmap->asce);
 653                 break;
 654         }
 655         default:
 656                 ret = -ENXIO;
 657                 break;
 658         }
 659         return ret;
 660 }
 661
 662 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
 663
 664 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
 665 {
 666         struct kvm_vcpu *vcpu;
 667         int i;
 668
 669         if (!test_kvm_facility(kvm, 76))
 670                 return -EINVAL;
 671
 672         mutex_lock(&kvm->lock);
 673         switch (attr->attr) {
 674         case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
 675                 get_random_bytes(
 676                         kvm->arch.crypto.crycb->aes_wrapping_key_mask,
 677                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 678                 kvm->arch.crypto.aes_kw = 1;
 679                 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
 680                 break;
 681         case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
 682                 get_random_bytes(
 683                         kvm->arch.crypto.crycb->dea_wrapping_key_mask,
 684                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 685                 kvm->arch.crypto.dea_kw = 1;
 686                 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
 687                 break;
 688         case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
 689                 kvm->arch.crypto.aes_kw = 0;
 690                 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
 691                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 692                 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
 693                 break;
 694         case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
 695                 kvm->arch.crypto.dea_kw = 0;
 696                 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
 697                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 698                 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
 699                 break;
 700         default:
 701                 mutex_unlock(&kvm->lock);
 702                 return -ENXIO;
 703         }
 704
 705         kvm_for_each_vcpu(i, vcpu, kvm) {
 706                 kvm_s390_vcpu_crypto_setup(vcpu);
 707                 exit_sie(vcpu);
 708         }
 709         mutex_unlock(&kvm->lock);
 710         return 0;
 711 }
 712
 713 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 714 {
 715         u8 gtod_high;
 716
 717         if (copy_from_user(&gtod_high, (void __user *)attr->addr,
 718                                            sizeof(gtod_high)))
 719                 return -EFAULT;
 720
 721         if (gtod_high != 0)
 722                 return -EINVAL;
 723         VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
 724
 725         return 0;
 726 }
 727
 728 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
 729 {
 730         u64 gtod;
 731
 732         if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
 733                 return -EFAULT;
 734
 735         kvm_s390_set_tod_clock(kvm, gtod);
 736         VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
 737         return 0;
 738 }
 739
 740 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
 741 {
 742         int ret;
 743
 744         if (attr->flags)
 745                 return -EINVAL;
 746
 747         switch (attr->attr) {
 748         case KVM_S390_VM_TOD_HIGH:
 749                 ret = kvm_s390_set_tod_high(kvm, attr);
 750                 break;
 751         case KVM_S390_VM_TOD_LOW:
 752                 ret = kvm_s390_set_tod_low(kvm, attr);
 753                 break;
 754         default:
 755                 ret = -ENXIO;
 756                 break;
 757         }
 758         return ret;
 759 }
 760
 761 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 762 {
 763         u8 gtod_high = 0;
 764
 765         if (copy_to_user((void __user *)attr->addr, &gtod_high,
 766                                          sizeof(gtod_high)))
 767                 return -EFAULT;
 768         VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
 769
 770         return 0;
 771 }
 772
 773 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
 774 {
 775         u64 gtod;
 776
 777         gtod = kvm_s390_get_tod_clock_fast(kvm);
 778         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
 779                 return -EFAULT;
 780         VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
 781
 782         return 0;
 783 }
 784
 785 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
 786 {
 787         int ret;
 788
 789         if (attr->flags)
 790                 return -EINVAL;
 791
 792         switch (attr->attr) {
 793         case KVM_S390_VM_TOD_HIGH:
 794                 ret = kvm_s390_get_tod_high(kvm, attr);
 795                 break;
 796         case KVM_S390_VM_TOD_LOW:
 797                 ret = kvm_s390_get_tod_low(kvm, attr);
 798                 break;
 799         default:
 800                 ret = -ENXIO;
 801                 break;
 802         }
 803         return ret;
 804 }
 805
 806 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
 807 {
 808         struct kvm_s390_vm_cpu_processor *proc;
 809         u16 lowest_ibc, unblocked_ibc;
 810         int ret = 0;
 811
 812         mutex_lock(&kvm->lock);
 813         if (kvm->created_vcpus) {
 814                 ret = -EBUSY;
 815                 goto out;
 816         }
 817         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
 818         if (!proc) {
 819                 ret = -ENOMEM;
 820                 goto out;
 821         }
 822         if (!copy_from_user(proc, (void __user *)attr->addr,
 823                             sizeof(*proc))) {
 824                 kvm->arch.model.cpuid = proc->cpuid;
 825                 lowest_ibc = sclp.ibc >> 16 & 0xfff;
 826                 unblocked_ibc = sclp.ibc & 0xfff;
 827                 if (lowest_ibc && proc->ibc) {
 828                         if (proc->ibc > unblocked_ibc)
 829                                 kvm->arch.model.ibc = unblocked_ibc;
 830                         else if (proc->ibc < lowest_ibc)
 831                                 kvm->arch.model.ibc = lowest_ibc;
 832                         else
 833                                 kvm->arch.model.ibc = proc->ibc;
 834                 }
 835                 memcpy(kvm->arch.model.fac_list, proc->fac_list,
 836                        S390_ARCH_FAC_LIST_SIZE_BYTE);
 837                 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
 838                          kvm->arch.model.ibc,
 839                          kvm->arch.model.cpuid);
 840                 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
 841                          kvm->arch.model.fac_list[0],
 842                          kvm->arch.model.fac_list[1],
 843                          kvm->arch.model.fac_list[2]);
 844         } else
 845                 ret = -EFAULT;
 846         kfree(proc);
 847 out:
 848         mutex_unlock(&kvm->lock);
 849         return ret;
 850 }
 851
 852 static int kvm_s390_set_processor_feat(struct kvm *kvm,
 853                                        struct kvm_device_attr *attr)
 854 {
 855         struct kvm_s390_vm_cpu_feat data;
 856         int ret = -EBUSY;
 857
 858         if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
 859                 return -EFAULT;
 860         if (!bitmap_subset((unsigned long *) data.feat,
 861                            kvm_s390_available_cpu_feat,
 862                            KVM_S390_VM_CPU_FEAT_NR_BITS))
 863                 return -EINVAL;
 864
 865         mutex_lock(&kvm->lock);
 866         if (!atomic_read(&kvm->online_vcpus)) {
 867                 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
 868                             KVM_S390_VM_CPU_FEAT_NR_BITS);
 869                 ret = 0;
 870         }
 871         mutex_unlock(&kvm->lock);
 872         return ret;
 873 }
 874
 875 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
 876                                           struct kvm_device_attr *attr)
 877 {
 878         /*
 879          * Once supported by kernel + hw, we have to store the subfunctions
 880          * in kvm->arch and remember that user space configured them.
 881          */
 882         return -ENXIO;
 883 }
 884
 885 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
 886 {
 887         int ret = -ENXIO;
 888
 889         switch (attr->attr) {
 890         case KVM_S390_VM_CPU_PROCESSOR:
 891                 ret = kvm_s390_set_processor(kvm, attr);
 892                 break;
 893         case KVM_S390_VM_CPU_PROCESSOR_FEAT:
 894                 ret = kvm_s390_set_processor_feat(kvm, attr);
 895                 break;
 896         case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
 897                 ret = kvm_s390_set_processor_subfunc(kvm, attr);
 898                 break;
 899         }
 900         return ret;
 901 }
 902
 903 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
 904 {
 905         struct kvm_s390_vm_cpu_processor *proc;
 906         int ret = 0;
 907
 908         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
 909         if (!proc) {
 910                 ret = -ENOMEM;
 911                 goto out;
 912         }
 913         proc->cpuid = kvm->arch.model.cpuid;
 914         proc->ibc = kvm->arch.model.ibc;
 915         memcpy(&proc->fac_list, kvm->arch.model.fac_list,
 916                S390_ARCH_FAC_LIST_SIZE_BYTE);
 917         VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
 918                  kvm->arch.model.ibc,
 919                  kvm->arch.model.cpuid);
 920         VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
 921                  kvm->arch.model.fac_list[0],
 922                  kvm->arch.model.fac_list[1],
 923                  kvm->arch.model.fac_list[2]);
 924         if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
 925                 ret = -EFAULT;
 926         kfree(proc);
 927 out:
 928         return ret;
 929 }
 930
 931 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
 932 {
 933         struct kvm_s390_vm_cpu_machine *mach;
 934         int ret = 0;
 935
 936         mach = kzalloc(sizeof(*mach), GFP_KERNEL);
 937         if (!mach) {
 938                 ret = -ENOMEM;
 939                 goto out;
 940         }
 941         get_cpu_id((struct cpuid *) &mach->cpuid);
 942         mach->ibc = sclp.ibc;
 943         memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
 944                S390_ARCH_FAC_LIST_SIZE_BYTE);
 945         memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
 946                sizeof(S390_lowcore.stfle_fac_list));
 947         VM_EVENT(kvm, 3, "GET: host ibc:  0x%4.4x, host cpuid:  0x%16.16llx",
 948                  kvm->arch.model.ibc,
 949                  kvm->arch.model.cpuid);
 950         VM_EVENT(kvm, 3, "GET: host facmask:  0x%16.16llx.%16.16llx.%16.16llx",
 951                  mach->fac_mask[0],
 952                  mach->fac_mask[1],
 953                  mach->fac_mask[2]);
 954         VM_EVENT(kvm, 3, "GET: host faclist:  0x%16.16llx.%16.16llx.%16.16llx",
 955                  mach->fac_list[0],
 956                  mach->fac_list[1],
 957                  mach->fac_list[2]);
 958         if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
 959                 ret = -EFAULT;
 960         kfree(mach);
 961 out:
 962         return ret;
 963 }
 964
 965 static int kvm_s390_get_processor_feat(struct kvm *kvm,
 966                                        struct kvm_device_attr *attr)
 967 {
 968         struct kvm_s390_vm_cpu_feat data;
 969
 970         bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
 971                     KVM_S390_VM_CPU_FEAT_NR_BITS);
 972         if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
 973                 return -EFAULT;
 974         return 0;
 975 }
 976
 977 static int kvm_s390_get_machine_feat(struct kvm *kvm,
 978                                      struct kvm_device_attr *attr)
 979 {
 980         struct kvm_s390_vm_cpu_feat data;
 981
 982         bitmap_copy((unsigned long *) data.feat,
 983                     kvm_s390_available_cpu_feat,
 984                     KVM_S390_VM_CPU_FEAT_NR_BITS);
 985         if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
 986                 return -EFAULT;
 987         return 0;
 988 }
 989
 990 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
 991                                           struct kvm_device_attr *attr)
 992 {
 993         /*
 994          * Once we can actually configure subfunctions (kernel + hw support),
 995          * we have to check if they were already set by user space, if so copy
 996          * them from kvm->arch.
 997          */
 998         return -ENXIO;
 999 }
1000
1001 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1002                                         struct kvm_device_attr *attr)
1003 {
1004         if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1005             sizeof(struct kvm_s390_vm_cpu_subfunc)))
1006                 return -EFAULT;
1007         return 0;
1008 }
1009 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1010 {
1011         int ret = -ENXIO;
1012
1013         switch (attr->attr) {
1014         case KVM_S390_VM_CPU_PROCESSOR:
1015                 ret = kvm_s390_get_processor(kvm, attr);
1016                 break;
1017         case KVM_S390_VM_CPU_MACHINE:
1018                 ret = kvm_s390_get_machine(kvm, attr);
1019                 break;
1020         case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1021                 ret = kvm_s390_get_processor_feat(kvm, attr);
1022                 break;
1023         case KVM_S390_VM_CPU_MACHINE_FEAT:
1024                 ret = kvm_s390_get_machine_feat(kvm, attr);
1025                 break;
1026         case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1027                 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1028                 break;
1029         case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1030                 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1031                 break;
1032         }
1033         return ret;
1034 }
1035
1036 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1037 {
1038         int ret;
1039
1040         switch (attr->group) {
1041         case KVM_S390_VM_MEM_CTRL:
1042                 ret = kvm_s390_set_mem_control(kvm, attr);
1043                 break;
1044         case KVM_S390_VM_TOD:
1045                 ret = kvm_s390_set_tod(kvm, attr);
1046                 break;
1047         case KVM_S390_VM_CPU_MODEL:
1048                 ret = kvm_s390_set_cpu_model(kvm, attr);
1049                 break;
1050         case KVM_S390_VM_CRYPTO:
1051                 ret = kvm_s390_vm_set_crypto(kvm, attr);
1052                 break;
1053         default:
1054                 ret = -ENXIO;
1055                 break;
1056         }
1057
1058         return ret;
1059 }
1060
1061 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1062 {
1063         int ret;
1064
1065         switch (attr->group) {
1066         case KVM_S390_VM_MEM_CTRL:
1067                 ret = kvm_s390_get_mem_control(kvm, attr);
1068                 break;
1069         case KVM_S390_VM_TOD:
1070                 ret = kvm_s390_get_tod(kvm, attr);
1071                 break;
1072         case KVM_S390_VM_CPU_MODEL:
1073                 ret = kvm_s390_get_cpu_model(kvm, attr);
1074                 break;
1075         default:
1076                 ret = -ENXIO;
1077                 break;
1078         }
1079
1080         return ret;
1081 }
1082
1083 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1084 {
1085         int ret;
1086
1087         switch (attr->group) {
1088         case KVM_S390_VM_MEM_CTRL:
1089                 switch (attr->attr) {
1090                 case KVM_S390_VM_MEM_ENABLE_CMMA:
1091                 case KVM_S390_VM_MEM_CLR_CMMA:
1092                         ret = sclp.has_cmma ? 0 : -ENXIO;
1093                         break;
1094                 case KVM_S390_VM_MEM_LIMIT_SIZE:
1095                         ret = 0;
1096                         break;
1097                 default:
1098                         ret = -ENXIO;
1099                         break;
1100                 }
1101                 break;
1102         case KVM_S390_VM_TOD:
1103                 switch (attr->attr) {
1104                 case KVM_S390_VM_TOD_LOW:
1105                 case KVM_S390_VM_TOD_HIGH:
1106                         ret = 0;
1107                         break;
1108                 default:
1109                         ret = -ENXIO;
1110                         break;
1111                 }
1112                 break;
1113         case KVM_S390_VM_CPU_MODEL:
1114                 switch (attr->attr) {
1115                 case KVM_S390_VM_CPU_PROCESSOR:
1116                 case KVM_S390_VM_CPU_MACHINE:
1117                 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1118                 case KVM_S390_VM_CPU_MACHINE_FEAT:
1119                 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1120                         ret = 0;
1121                         break;
1122                 /* configuring subfunctions is not supported yet */
1123                 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1124                 default:
1125                         ret = -ENXIO;
1126                         break;
1127                 }
1128                 break;
1129         case KVM_S390_VM_CRYPTO:
1130                 switch (attr->attr) {
1131                 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1132                 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1133                 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1134                 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1135                         ret = 0;
1136                         break;
1137                 default:
1138                         ret = -ENXIO;
1139                         break;
1140                 }
1141                 break;
1142         default:
1143                 ret = -ENXIO;
1144                 break;
1145         }
1146
1147         return ret;
1148 }
1149
1150 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1151 {
1152         uint8_t *keys;
1153         uint64_t hva;
1154         int i, r = 0;
1155
1156         if (args->flags != 0)
1157                 return -EINVAL;
1158
1159         /* Is this guest using storage keys? */
1160         if (!mm_use_skey(current->mm))
1161                 return KVM_S390_GET_SKEYS_NONE;
1162
1163         /* Enforce sane limit on memory allocation */
1164         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1165                 return -EINVAL;
1166
1167         keys = kmalloc_array(args->count, sizeof(uint8_t),
1168                              GFP_KERNEL | __GFP_NOWARN);
1169         if (!keys)
1170                 keys = vmalloc(sizeof(uint8_t) * args->count);
1171         if (!keys)
1172                 return -ENOMEM;
1173
1174         down_read(&current->mm->mmap_sem);
1175         for (i = 0; i < args->count; i++) {
1176                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1177                 if (kvm_is_error_hva(hva)) {
1178                         r = -EFAULT;
1179                         break;
1180                 }
1181
1182                 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1183                 if (r)
1184                         break;
1185         }
1186         up_read(&current->mm->mmap_sem);
1187
1188         if (!r) {
1189                 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1190                                  sizeof(uint8_t) * args->count);
1191                 if (r)
1192                         r = -EFAULT;
1193         }
1194
1195         kvfree(keys);
1196         return r;
1197 }
1198
1199 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1200 {
1201         uint8_t *keys;
1202         uint64_t hva;
1203         int i, r = 0;
1204
1205         if (args->flags != 0)
1206                 return -EINVAL;
1207
1208         /* Enforce sane limit on memory allocation */
1209         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1210                 return -EINVAL;
1211
1212         keys = kmalloc_array(args->count, sizeof(uint8_t),
1213                              GFP_KERNEL | __GFP_NOWARN);
1214         if (!keys)
1215                 keys = vmalloc(sizeof(uint8_t) * args->count);
1216         if (!keys)
1217                 return -ENOMEM;
1218
1219         r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1220                            sizeof(uint8_t) * args->count);
1221         if (r) {
1222                 r = -EFAULT;
1223                 goto out;
1224         }
1225
1226         /* Enable storage key handling for the guest */
1227         r = s390_enable_skey();
1228         if (r)
1229                 goto out;
1230
1231         down_read(&current->mm->mmap_sem);
1232         for (i = 0; i < args->count; i++) {
1233                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1234                 if (kvm_is_error_hva(hva)) {
1235                         r = -EFAULT;
1236                         break;
1237                 }
1238
1239                 /* Lowest order bit is reserved */
1240                 if (keys[i] & 0x01) {
1241                         r = -EINVAL;
1242                         break;
1243                 }
1244
1245                 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1246                 if (r)
1247                         break;
1248         }
1249         up_read(&current->mm->mmap_sem);
1250 out:
1251         kvfree(keys);
1252         return r;
1253 }
1254
1255 long kvm_arch_vm_ioctl(struct file *filp,
1256                        unsigned int ioctl, unsigned long arg)
1257 {
1258         struct kvm *kvm = filp->private_data;
1259         void __user *argp = (void __user *)arg;
1260         struct kvm_device_attr attr;
1261         int r;
1262
1263         switch (ioctl) {
1264         case KVM_S390_INTERRUPT: {
1265                 struct kvm_s390_interrupt s390int;
1266
1267                 r = -EFAULT;
1268                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1269                         break;
1270                 r = kvm_s390_inject_vm(kvm, &s390int);
1271                 break;
1272         }
1273         case KVM_ENABLE_CAP: {
1274                 struct kvm_enable_cap cap;
1275                 r = -EFAULT;
1276                 if (copy_from_user(&cap, argp, sizeof(cap)))
1277                         break;
1278                 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1279                 break;
1280         }
1281         case KVM_CREATE_IRQCHIP: {
1282                 struct kvm_irq_routing_entry routing;
1283
1284                 r = -EINVAL;
1285                 if (kvm->arch.use_irqchip) {
1286                         /* Set up dummy routing. */
1287                         memset(&routing, 0, sizeof(routing));
1288                         r = kvm_set_irq_routing(kvm, &routing, 0, 0);
1289                 }
1290                 break;
1291         }
1292         case KVM_SET_DEVICE_ATTR: {
1293                 r = -EFAULT;
1294                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1295                         break;
1296                 r = kvm_s390_vm_set_attr(kvm, &attr);
1297                 break;
1298         }
1299         case KVM_GET_DEVICE_ATTR: {
1300                 r = -EFAULT;
1301                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1302                         break;
1303                 r = kvm_s390_vm_get_attr(kvm, &attr);
1304                 break;
1305         }
1306         case KVM_HAS_DEVICE_ATTR: {
1307                 r = -EFAULT;
1308                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1309                         break;
1310                 r = kvm_s390_vm_has_attr(kvm, &attr);
1311                 break;
1312         }
1313         case KVM_S390_GET_SKEYS: {
1314                 struct kvm_s390_skeys args;
1315
1316                 r = -EFAULT;
1317                 if (copy_from_user(&args, argp,
1318                                    sizeof(struct kvm_s390_skeys)))
1319                         break;
1320                 r = kvm_s390_get_skeys(kvm, &args);
1321                 break;
1322         }
1323         case KVM_S390_SET_SKEYS: {
1324                 struct kvm_s390_skeys args;
1325
1326                 r = -EFAULT;
1327                 if (copy_from_user(&args, argp,
1328                                    sizeof(struct kvm_s390_skeys)))
1329                         break;
1330                 r = kvm_s390_set_skeys(kvm, &args);
1331                 break;
1332         }
1333         default:
1334                 r = -ENOTTY;
1335         }
1336
1337         return r;
1338 }
1339
1340 static int kvm_s390_query_ap_config(u8 *config)
1341 {
1342         u32 fcn_code = 0x04000000UL;
1343         u32 cc = 0;
1344
1345         memset(config, 0, 128);
1346         asm volatile(
1347                 "lgr 0,%1\n"
1348                 "lgr 2,%2\n"
1349                 ".long 0xb2af0000\n"            /* PQAP(QCI) */
1350                 "0: ipm %0\n"
1351                 "srl %0,28\n"
1352                 "1:\n"
1353                 EX_TABLE(0b, 1b)
1354                 : "+r" (cc)
1355                 : "r" (fcn_code), "r" (config)
1356                 : "cc", "0", "2", "memory"
1357         );
1358
1359         return cc;
1360 }
1361
1362 static int kvm_s390_apxa_installed(void)
1363 {
1364         u8 config[128];
1365         int cc;
1366
1367         if (test_facility(12)) {
1368                 cc = kvm_s390_query_ap_config(config);
1369
1370                 if (cc)
1371                         pr_err("PQAP(QCI) failed with cc=%d", cc);
1372                 else
1373                         return config[0] & 0x40;
1374         }
1375
1376         return 0;
1377 }
1378
1379 static void kvm_s390_set_crycb_format(struct kvm *kvm)
1380 {
1381         kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1382
1383         if (kvm_s390_apxa_installed())
1384                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1385         else
1386                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1387 }
1388
1389 static u64 kvm_s390_get_initial_cpuid(void)
1390 {
1391         struct cpuid cpuid;
1392
1393         get_cpu_id(&cpuid);
1394         cpuid.version = 0xff;
1395         return *((u64 *) &cpuid);
1396 }
1397
1398 static void kvm_s390_crypto_init(struct kvm *kvm)
1399 {
1400         if (!test_kvm_facility(kvm, 76))
1401                 return;
1402
1403         kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
1404         kvm_s390_set_crycb_format(kvm);
1405
1406         /* Enable AES/DEA protected key functions by default */
1407         kvm->arch.crypto.aes_kw = 1;
1408         kvm->arch.crypto.dea_kw = 1;
1409         get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1410                          sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1411         get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1412                          sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1413 }
1414
1415 static void sca_dispose(struct kvm *kvm)
1416 {
1417         if (kvm->arch.use_esca)
1418                 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
1419         else
1420                 free_page((unsigned long)(kvm->arch.sca));
1421         kvm->arch.sca = NULL;
1422 }
1423
1424 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1425 {
1426         gfp_t alloc_flags = GFP_KERNEL;
1427         int i, rc;
1428         char debug_name[16];
1429         static unsigned long sca_offset;
1430
1431         rc = -EINVAL;
1432 #ifdef CONFIG_KVM_S390_UCONTROL
1433         if (type & ~KVM_VM_S390_UCONTROL)
1434                 goto out_err;
1435         if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1436                 goto out_err;
1437 #else
1438         if (type)
1439                 goto out_err;
1440 #endif
1441
1442         rc = s390_enable_sie();
1443         if (rc)
1444                 goto out_err;
1445
1446         rc = -ENOMEM;
1447
1448         ratelimit_state_init(&kvm->arch.sthyi_limit, 5 * HZ, 500);
1449
1450         kvm->arch.use_esca = 0; /* start with basic SCA */
1451         if (!sclp.has_64bscao)
1452                 alloc_flags |= GFP_DMA;
1453         rwlock_init(&kvm->arch.sca_lock);
1454         kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
1455         if (!kvm->arch.sca)
1456                 goto out_err;
1457         spin_lock(&kvm_lock);
1458         sca_offset += 16;
1459         if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
1460                 sca_offset = 0;
1461         kvm->arch.sca = (struct bsca_block *)
1462                         ((char *) kvm->arch.sca + sca_offset);
1463         spin_unlock(&kvm_lock);
1464
1465         sprintf(debug_name, "kvm-%u", current->pid);
1466
1467         kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
1468         if (!kvm->arch.dbf)
1469                 goto out_err;
1470
1471         kvm->arch.sie_page2 =
1472              (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1473         if (!kvm->arch.sie_page2)
1474                 goto out_err;
1475
1476         /* Populate the facility mask initially. */
1477         memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
1478                sizeof(S390_lowcore.stfle_fac_list));
1479         for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1480                 if (i < kvm_s390_fac_list_mask_size())
1481                         kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
1482                 else
1483                         kvm->arch.model.fac_mask[i] = 0UL;
1484         }
1485
1486         /* Populate the facility list initially. */
1487         kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1488         memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
1489                S390_ARCH_FAC_LIST_SIZE_BYTE);
1490
1491         set_kvm_facility(kvm->arch.model.fac_mask, 74);
1492         set_kvm_facility(kvm->arch.model.fac_list, 74);
1493
1494         kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
1495         kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1496
1497         kvm_s390_crypto_init(kvm);
1498
1499         spin_lock_init(&kvm->arch.float_int.lock);
1500         for (i = 0; i < FIRQ_LIST_COUNT; i++)
1501                 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1502         init_waitqueue_head(&kvm->arch.ipte_wq);
1503         mutex_init(&kvm->arch.ipte_mutex);
1504
1505         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1506         VM_EVENT(kvm, 3, "vm created with type %lu", type);
1507
1508         if (type & KVM_VM_S390_UCONTROL) {
1509                 kvm->arch.gmap = NULL;
1510                 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
1511         } else {
1512                 if (sclp.hamax == U64_MAX)
1513                         kvm->arch.mem_limit = TASK_MAX_SIZE;
1514                 else
1515                         kvm->arch.mem_limit = min_t(unsigned long, TASK_MAX_SIZE,
1516                                                     sclp.hamax + 1);
1517                 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
1518                 if (!kvm->arch.gmap)
1519                         goto out_err;
1520                 kvm->arch.gmap->private = kvm;
1521                 kvm->arch.gmap->pfault_enabled = 0;
1522         }
1523
1524         kvm->arch.css_support = 0;
1525         kvm->arch.use_irqchip = 0;
1526         kvm->arch.epoch = 0;
1527
1528         spin_lock_init(&kvm->arch.start_stop_lock);
1529         kvm_s390_vsie_init(kvm);
1530         KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
1531
1532         return 0;
1533 out_err:
1534         free_page((unsigned long)kvm->arch.sie_page2);
1535         debug_unregister(kvm->arch.dbf);
1536         sca_dispose(kvm);
1537         KVM_EVENT(3, "creation of vm failed: %d", rc);
1538         return rc;
1539 }
1540
1541 bool kvm_arch_has_vcpu_debugfs(void)
1542 {
1543         return false;
1544 }
1545
1546 int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
1547 {
1548         return 0;
1549 }
1550
1551 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1552 {
1553         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1554         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1555         kvm_s390_clear_local_irqs(vcpu);
1556         kvm_clear_async_pf_completion_queue(vcpu);
1557         if (!kvm_is_ucontrol(vcpu->kvm))
1558                 sca_del_vcpu(vcpu);
1559
1560         if (kvm_is_ucontrol(vcpu->kvm))
1561                 gmap_remove(vcpu->arch.gmap);
1562
1563         if (vcpu->kvm->arch.use_cmma)
1564                 kvm_s390_vcpu_unsetup_cmma(vcpu);
1565         free_page((unsigned long)(vcpu->arch.sie_block));
1566
1567         kvm_vcpu_uninit(vcpu);
1568         kmem_cache_free(kvm_vcpu_cache, vcpu);
1569 }
1570
1571 static void kvm_free_vcpus(struct kvm *kvm)
1572 {
1573         unsigned int i;
1574         struct kvm_vcpu *vcpu;
1575
1576         kvm_for_each_vcpu(i, vcpu, kvm)
1577                 kvm_arch_vcpu_destroy(vcpu);
1578
1579         mutex_lock(&kvm->lock);
1580         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1581                 kvm->vcpus[i] = NULL;
1582
1583         atomic_set(&kvm->online_vcpus, 0);
1584         mutex_unlock(&kvm->lock);
1585 }
1586
1587 void kvm_arch_destroy_vm(struct kvm *kvm)
1588 {
1589         kvm_free_vcpus(kvm);
1590         sca_dispose(kvm);
1591         debug_unregister(kvm->arch.dbf);
1592         free_page((unsigned long)kvm->arch.sie_page2);
1593         if (!kvm_is_ucontrol(kvm))
1594                 gmap_remove(kvm->arch.gmap);
1595         kvm_s390_destroy_adapters(kvm);
1596         kvm_s390_clear_float_irqs(kvm);
1597         kvm_s390_vsie_destroy(kvm);
1598         KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
1599 }
1600
1601 /* Section: vcpu related */
1602 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1603 {
1604         vcpu->arch.gmap = gmap_create(current->mm, -1UL);
1605         if (!vcpu->arch.gmap)
1606                 return -ENOMEM;
1607         vcpu->arch.gmap->private = vcpu->kvm;
1608
1609         return 0;
1610 }
1611
1612 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1613 {
1614         if (!kvm_s390_use_sca_entries())
1615                 return;
1616         read_lock(&vcpu->kvm->arch.sca_lock);
1617         if (vcpu->kvm->arch.use_esca) {
1618                 struct esca_block *sca = vcpu->kvm->arch.sca;
1619
1620                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1621                 sca->cpu[vcpu->vcpu_id].sda = 0;
1622         } else {
1623                 struct bsca_block *sca = vcpu->kvm->arch.sca;
1624
1625                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1626                 sca->cpu[vcpu->vcpu_id].sda = 0;
1627         }
1628         read_unlock(&vcpu->kvm->arch.sca_lock);
1629 }
1630
1631 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
1632 {
1633         if (!kvm_s390_use_sca_entries()) {
1634                 struct bsca_block *sca = vcpu->kvm->arch.sca;
1635
1636                 /* we still need the basic sca for the ipte control */
1637                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1638                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
1639         }
1640         read_lock(&vcpu->kvm->arch.sca_lock);
1641         if (vcpu->kvm->arch.use_esca) {
1642                 struct esca_block *sca = vcpu->kvm->arch.sca;
1643
1644                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1645                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1646                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
1647                 vcpu->arch.sie_block->ecb2 |= 0x04U;
1648                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1649         } else {
1650                 struct bsca_block *sca = vcpu->kvm->arch.sca;
1651
1652                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1653                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1654                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
1655                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1656         }
1657         read_unlock(&vcpu->kvm->arch.sca_lock);
1658 }
1659
1660 /* Basic SCA to Extended SCA data copy routines */
1661 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1662 {
1663         d->sda = s->sda;
1664         d->sigp_ctrl.c = s->sigp_ctrl.c;
1665         d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1666 }
1667
1668 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1669 {
1670         int i;
1671
1672         d->ipte_control = s->ipte_control;
1673         d->mcn[0] = s->mcn;
1674         for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1675                 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1676 }
1677
1678 static int sca_switch_to_extended(struct kvm *kvm)
1679 {
1680         struct bsca_block *old_sca = kvm->arch.sca;
1681         struct esca_block *new_sca;
1682         struct kvm_vcpu *vcpu;
1683         unsigned int vcpu_idx;
1684         u32 scaol, scaoh;
1685
1686         new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1687         if (!new_sca)
1688                 return -ENOMEM;
1689
1690         scaoh = (u32)((u64)(new_sca) >> 32);
1691         scaol = (u32)(u64)(new_sca) & ~0x3fU;
1692
1693         kvm_s390_vcpu_block_all(kvm);
1694         write_lock(&kvm->arch.sca_lock);
1695
1696         sca_copy_b_to_e(new_sca, old_sca);
1697
1698         kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1699                 vcpu->arch.sie_block->scaoh = scaoh;
1700                 vcpu->arch.sie_block->scaol = scaol;
1701                 vcpu->arch.sie_block->ecb2 |= 0x04U;
1702         }
1703         kvm->arch.sca = new_sca;
1704         kvm->arch.use_esca = 1;
1705
1706         write_unlock(&kvm->arch.sca_lock);
1707         kvm_s390_vcpu_unblock_all(kvm);
1708
1709         free_page((unsigned long)old_sca);
1710
1711         VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1712                  old_sca, kvm->arch.sca);
1713         return 0;
1714 }
1715
1716 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1717 {
1718         int rc;
1719
1720         if (!kvm_s390_use_sca_entries()) {
1721                 if (id < KVM_MAX_VCPUS)
1722                         return true;
1723                 return false;
1724         }
1725         if (id < KVM_S390_BSCA_CPU_SLOTS)
1726                 return true;
1727         if (!sclp.has_esca || !sclp.has_64bscao)
1728                 return false;
1729
1730         mutex_lock(&kvm->lock);
1731         rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1732         mutex_unlock(&kvm->lock);
1733
1734         return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
1735 }
1736
1737 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1738 {
1739         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1740         kvm_clear_async_pf_completion_queue(vcpu);
1741         vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1742                                     KVM_SYNC_GPRS |
1743                                     KVM_SYNC_ACRS |
1744                                     KVM_SYNC_CRS |
1745                                     KVM_SYNC_ARCH0 |
1746                                     KVM_SYNC_PFAULT;
1747         kvm_s390_set_prefix(vcpu, 0);
1748         if (test_kvm_facility(vcpu->kvm, 64))
1749                 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
1750         /* fprs can be synchronized via vrs, even if the guest has no vx. With
1751          * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1752          */
1753         if (MACHINE_HAS_VX)
1754                 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1755         else
1756                 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
1757
1758         if (kvm_is_ucontrol(vcpu->kvm))
1759                 return __kvm_ucontrol_vcpu_init(vcpu);
1760
1761         return 0;
1762 }
1763
1764 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1765 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1766 {
1767         WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
1768         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1769         vcpu->arch.cputm_start = get_tod_clock_fast();
1770         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1771 }
1772
1773 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1774 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1775 {
1776         WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
1777         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1778         vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1779         vcpu->arch.cputm_start = 0;
1780         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1781 }
1782
1783 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1784 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1785 {
1786         WARN_ON_ONCE(vcpu->arch.cputm_enabled);
1787         vcpu->arch.cputm_enabled = true;
1788         __start_cpu_timer_accounting(vcpu);
1789 }
1790
1791 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1792 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1793 {
1794         WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
1795         __stop_cpu_timer_accounting(vcpu);
1796         vcpu->arch.cputm_enabled = false;
1797 }
1798
1799 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1800 {
1801         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1802         __enable_cpu_timer_accounting(vcpu);
1803         preempt_enable();
1804 }
1805
1806 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1807 {
1808         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1809         __disable_cpu_timer_accounting(vcpu);
1810         preempt_enable();
1811 }
1812
1813 /* set the cpu timer - may only be called from the VCPU thread itself */
1814 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
1815 {
1816         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1817         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1818         if (vcpu->arch.cputm_enabled)
1819                 vcpu->arch.cputm_start = get_tod_clock_fast();
1820         vcpu->arch.sie_block->cputm = cputm;
1821         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1822         preempt_enable();
1823 }
1824
1825 /* update and get the cpu timer - can also be called from other VCPU threads */
1826 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
1827 {
1828         unsigned int seq;
1829         __u64 value;
1830
1831         if (unlikely(!vcpu->arch.cputm_enabled))
1832                 return vcpu->arch.sie_block->cputm;
1833
1834         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1835         do {
1836                 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
1837                 /*
1838                  * If the writer would ever execute a read in the critical
1839                  * section, e.g. in irq context, we have a deadlock.
1840                  */
1841                 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
1842                 value = vcpu->arch.sie_block->cputm;
1843                 /* if cputm_start is 0, accounting is being started/stopped */
1844                 if (likely(vcpu->arch.cputm_start))
1845                         value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1846         } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
1847         preempt_enable();
1848         return value;
1849 }
1850
1851 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1852 {
1853
1854         gmap_enable(vcpu->arch.enabled_gmap);
1855         atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1856         if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
1857                 __start_cpu_timer_accounting(vcpu);
1858         vcpu->cpu = cpu;
1859 }
1860
1861 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1862 {
1863         vcpu->cpu = -1;
1864         if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
1865                 __stop_cpu_timer_accounting(vcpu);
1866         atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1867         vcpu->arch.enabled_gmap = gmap_get_enabled();
1868         gmap_disable(vcpu->arch.enabled_gmap);
1869
1870 }
1871
1872 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1873 {
1874         /* this equals initial cpu reset in pop, but we don't switch to ESA */
1875         vcpu->arch.sie_block->gpsw.mask = 0UL;
1876         vcpu->arch.sie_block->gpsw.addr = 0UL;
1877         kvm_s390_set_prefix(vcpu, 0);
1878         kvm_s390_set_cpu_timer(vcpu, 0);
1879         vcpu->arch.sie_block->ckc       = 0UL;
1880         vcpu->arch.sie_block->todpr     = 0;
1881         memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1882         vcpu->arch.sie_block->gcr[0]  = 0xE0UL;
1883         vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1884         /* make sure the new fpc will be lazily loaded */
1885         save_fpu_regs();
1886         current->thread.fpu.fpc = 0;
1887         vcpu->arch.sie_block->gbea = 1;
1888         vcpu->arch.sie_block->pp = 0;
1889         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1890         kvm_clear_async_pf_completion_queue(vcpu);
1891         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1892                 kvm_s390_vcpu_stop(vcpu);
1893         kvm_s390_clear_local_irqs(vcpu);
1894 }
1895
1896 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1897 {
1898         mutex_lock(&vcpu->kvm->lock);
1899         preempt_disable();
1900         vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1901         preempt_enable();
1902         mutex_unlock(&vcpu->kvm->lock);
1903         if (!kvm_is_ucontrol(vcpu->kvm)) {
1904                 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1905                 sca_add_vcpu(vcpu);
1906         }
1907         if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
1908                 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
1909         /* make vcpu_load load the right gmap on the first trigger */
1910         vcpu->arch.enabled_gmap = vcpu->arch.gmap;
1911 }
1912
1913 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1914 {
1915         if (!test_kvm_facility(vcpu->kvm, 76))
1916                 return;
1917
1918         vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1919
1920         if (vcpu->kvm->arch.crypto.aes_kw)
1921                 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1922         if (vcpu->kvm->arch.crypto.dea_kw)
1923                 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1924
1925         vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1926 }
1927
1928 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1929 {
1930         free_page(vcpu->arch.sie_block->cbrlo);
1931         vcpu->arch.sie_block->cbrlo = 0;
1932 }
1933
1934 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1935 {
1936         vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1937         if (!vcpu->arch.sie_block->cbrlo)
1938                 return -ENOMEM;
1939
1940         vcpu->arch.sie_block->ecb2 |= 0x80;
1941         vcpu->arch.sie_block->ecb2 &= ~0x08;
1942         return 0;
1943 }
1944
1945 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1946 {
1947         struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1948
1949         vcpu->arch.sie_block->ibc = model->ibc;
1950         if (test_kvm_facility(vcpu->kvm, 7))
1951                 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
1952 }
1953
1954 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1955 {
1956         int rc = 0;
1957
1958         atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1959                                                     CPUSTAT_SM |
1960                                                     CPUSTAT_STOPPED);
1961
1962         if (test_kvm_facility(vcpu->kvm, 78))
1963                 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
1964         else if (test_kvm_facility(vcpu->kvm, 8))
1965                 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
1966
1967         kvm_s390_vcpu_setup_model(vcpu);
1968
1969         /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
1970         if (MACHINE_HAS_ESOP)
1971                 vcpu->arch.sie_block->ecb |= 0x02;
1972         if (test_kvm_facility(vcpu->kvm, 9))
1973                 vcpu->arch.sie_block->ecb |= 0x04;
1974         if (test_kvm_facility(vcpu->kvm, 73))
1975                 vcpu->arch.sie_block->ecb |= 0x10;
1976
1977         if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
1978                 vcpu->arch.sie_block->ecb2 |= 0x08;
1979         if (test_kvm_facility(vcpu->kvm, 130))
1980                 vcpu->arch.sie_block->ecb2 |= 0x20;
1981         vcpu->arch.sie_block->eca = 0x1002000U;
1982         if (sclp.has_cei)
1983                 vcpu->arch.sie_block->eca |= 0x80000000U;
1984         if (sclp.has_ib)
1985                 vcpu->arch.sie_block->eca |= 0x40000000U;
1986         if (sclp.has_siif)
1987                 vcpu->arch.sie_block->eca |= 1;
1988         if (sclp.has_sigpif)
1989                 vcpu->arch.sie_block->eca |= 0x10000000U;
1990         if (test_kvm_facility(vcpu->kvm, 129)) {
1991                 vcpu->arch.sie_block->eca |= 0x00020000;
1992                 vcpu->arch.sie_block->ecd |= 0x20000000;
1993         }
1994         vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
1995         vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1996
1997         if (vcpu->kvm->arch.use_cmma) {
1998                 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1999                 if (rc)
2000                         return rc;
2001         }
2002         hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2003         vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
2004
2005         kvm_s390_vcpu_crypto_setup(vcpu);
2006
2007         return rc;
2008 }
2009
2010 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
2011                                       unsigned int id)
2012 {
2013         struct kvm_vcpu *vcpu;
2014         struct sie_page *sie_page;
2015         int rc = -EINVAL;
2016
2017         if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
2018                 goto out;
2019
2020         rc = -ENOMEM;
2021
2022         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
2023         if (!vcpu)
2024                 goto out;
2025
2026         sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
2027         if (!sie_page)
2028                 goto out_free_cpu;
2029
2030         vcpu->arch.sie_block = &sie_page->sie_block;
2031         vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
2032
2033         /* the real guest size will always be smaller than msl */
2034         vcpu->arch.sie_block->mso = 0;
2035         vcpu->arch.sie_block->msl = sclp.hamax;
2036
2037         vcpu->arch.sie_block->icpua = id;
2038         spin_lock_init(&vcpu->arch.local_int.lock);
2039         vcpu->arch.local_int.float_int = &kvm->arch.float_int;
2040         vcpu->arch.local_int.wq = &vcpu->wq;
2041         vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
2042         seqcount_init(&vcpu->arch.cputm_seqcount);
2043
2044         rc = kvm_vcpu_init(vcpu, kvm, id);
2045         if (rc)
2046                 goto out_free_sie_block;
2047         VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
2048                  vcpu->arch.sie_block);
2049         trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
2050
2051         return vcpu;
2052 out_free_sie_block:
2053         free_page((unsigned long)(vcpu->arch.sie_block));
2054 out_free_cpu:
2055         kmem_cache_free(kvm_vcpu_cache, vcpu);
2056 out:
2057         return ERR_PTR(rc);
2058 }
2059
2060 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
2061 {
2062         return kvm_s390_vcpu_has_irq(vcpu, 0);
2063 }
2064
2065 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
2066 {
2067         atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2068         exit_sie(vcpu);
2069 }
2070
2071 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
2072 {
2073         atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2074 }
2075
2076 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
2077 {
2078         atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2079         exit_sie(vcpu);
2080 }
2081
2082 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
2083 {
2084         atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2085 }
2086
2087 /*
2088  * Kick a guest cpu out of SIE and wait until SIE is not running.
2089  * If the CPU is not running (e.g. waiting as idle) the function will
2090  * return immediately. */
2091 void exit_sie(struct kvm_vcpu *vcpu)
2092 {
2093         atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
2094         while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
2095                 cpu_relax();
2096 }
2097
2098 /* Kick a guest cpu out of SIE to process a request synchronously */
2099 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
2100 {
2101         kvm_make_request(req, vcpu);
2102         kvm_s390_vcpu_request(vcpu);
2103 }
2104
2105 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
2106                               unsigned long end)
2107 {
2108         struct kvm *kvm = gmap->private;
2109         struct kvm_vcpu *vcpu;
2110         unsigned long prefix;
2111         int i;
2112
2113         if (gmap_is_shadow(gmap))
2114                 return;
2115         if (start >= 1UL << 31)
2116                 /* We are only interested in prefix pages */
2117                 return;
2118         kvm_for_each_vcpu(i, vcpu, kvm) {
2119                 /* match against both prefix pages */
2120                 prefix = kvm_s390_get_prefix(vcpu);
2121                 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
2122                         VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
2123                                    start, end);
2124                         kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2125                 }
2126         }
2127 }
2128
2129 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2130 {
2131         /* kvm common code refers to this, but never calls it */
2132         BUG();
2133         return 0;
2134 }
2135
2136 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2137                                            struct kvm_one_reg *reg)
2138 {
2139         int r = -EINVAL;
2140
2141         switch (reg->id) {
2142         case KVM_REG_S390_TODPR:
2143                 r = put_user(vcpu->arch.sie_block->todpr,
2144                              (u32 __user *)reg->addr);
2145                 break;
2146         case KVM_REG_S390_EPOCHDIFF:
2147                 r = put_user(vcpu->arch.sie_block->epoch,
2148                              (u64 __user *)reg->addr);
2149                 break;
2150         case KVM_REG_S390_CPU_TIMER:
2151                 r = put_user(kvm_s390_get_cpu_timer(vcpu),
2152                              (u64 __user *)reg->addr);
2153                 break;
2154         case KVM_REG_S390_CLOCK_COMP:
2155                 r = put_user(vcpu->arch.sie_block->ckc,
2156                              (u64 __user *)reg->addr);
2157                 break;
2158         case KVM_REG_S390_PFTOKEN:
2159                 r = put_user(vcpu->arch.pfault_token,
2160                              (u64 __user *)reg->addr);
2161                 break;
2162         case KVM_REG_S390_PFCOMPARE:
2163                 r = put_user(vcpu->arch.pfault_compare,
2164                              (u64 __user *)reg->addr);
2165                 break;
2166         case KVM_REG_S390_PFSELECT:
2167                 r = put_user(vcpu->arch.pfault_select,
2168                              (u64 __user *)reg->addr);
2169                 break;
2170         case KVM_REG_S390_PP:
2171                 r = put_user(vcpu->arch.sie_block->pp,
2172                              (u64 __user *)reg->addr);
2173                 break;
2174         case KVM_REG_S390_GBEA:
2175                 r = put_user(vcpu->arch.sie_block->gbea,
2176                              (u64 __user *)reg->addr);
2177                 break;
2178         default:
2179                 break;
2180         }
2181
2182         return r;
2183 }
2184
2185 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2186                                            struct kvm_one_reg *reg)
2187 {
2188         int r = -EINVAL;
2189         __u64 val;
2190
2191         switch (reg->id) {
2192         case KVM_REG_S390_TODPR:
2193                 r = get_user(vcpu->arch.sie_block->todpr,
2194                              (u32 __user *)reg->addr);
2195                 break;
2196         case KVM_REG_S390_EPOCHDIFF:
2197                 r = get_user(vcpu->arch.sie_block->epoch,
2198                              (u64 __user *)reg->addr);
2199                 break;
2200         case KVM_REG_S390_CPU_TIMER:
2201                 r = get_user(val, (u64 __user *)reg->addr);
2202                 if (!r)
2203                         kvm_s390_set_cpu_timer(vcpu, val);
2204                 break;
2205         case KVM_REG_S390_CLOCK_COMP:
2206                 r = get_user(vcpu->arch.sie_block->ckc,
2207                              (u64 __user *)reg->addr);
2208                 break;
2209         case KVM_REG_S390_PFTOKEN:
2210                 r = get_user(vcpu->arch.pfault_token,
2211                              (u64 __user *)reg->addr);
2212                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2213                         kvm_clear_async_pf_completion_queue(vcpu);
2214                 break;
2215         case KVM_REG_S390_PFCOMPARE:
2216                 r = get_user(vcpu->arch.pfault_compare,
2217                              (u64 __user *)reg->addr);
2218                 break;
2219         case KVM_REG_S390_PFSELECT:
2220                 r = get_user(vcpu->arch.pfault_select,
2221                              (u64 __user *)reg->addr);
2222                 break;
2223         case KVM_REG_S390_PP:
2224                 r = get_user(vcpu->arch.sie_block->pp,
2225                              (u64 __user *)reg->addr);
2226                 break;
2227         case KVM_REG_S390_GBEA:
2228                 r = get_user(vcpu->arch.sie_block->gbea,
2229                              (u64 __user *)reg->addr);
2230                 break;
2231         default:
2232                 break;
2233         }
2234
2235         return r;
2236 }
2237
2238 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2239 {
2240         kvm_s390_vcpu_initial_reset(vcpu);
2241         return 0;
2242 }
2243
2244 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2245 {
2246         memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
2247         return 0;
2248 }
2249
2250 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2251 {
2252         memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
2253         return 0;
2254 }
2255
2256 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2257                                   struct kvm_sregs *sregs)
2258 {
2259         memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
2260         memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
2261         return 0;
2262 }
2263
2264 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2265                                   struct kvm_sregs *sregs)
2266 {
2267         memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
2268         memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
2269         return 0;
2270 }
2271
2272 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2273 {
2274         if (test_fp_ctl(fpu->fpc))
2275                 return -EINVAL;
2276         vcpu->run->s.regs.fpc = fpu->fpc;
2277         if (MACHINE_HAS_VX)
2278                 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
2279                                  (freg_t *) fpu->fprs);
2280         else
2281                 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
2282         return 0;
2283 }
2284
2285 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2286 {
2287         /* make sure we have the latest values */
2288         save_fpu_regs();
2289         if (MACHINE_HAS_VX)
2290                 convert_vx_to_fp((freg_t *) fpu->fprs,
2291                                  (__vector128 *) vcpu->run->s.regs.vrs);
2292         else
2293                 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
2294         fpu->fpc = vcpu->run->s.regs.fpc;
2295         return 0;
2296 }
2297
2298 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2299 {
2300         int rc = 0;
2301
2302         if (!is_vcpu_stopped(vcpu))
2303                 rc = -EBUSY;
2304         else {
2305                 vcpu->run->psw_mask = psw.mask;
2306                 vcpu->run->psw_addr = psw.addr;
2307         }
2308         return rc;
2309 }
2310
2311 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2312                                   struct kvm_translation *tr)
2313 {
2314         return -EINVAL; /* not implemented yet */
2315 }
2316
2317 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2318                               KVM_GUESTDBG_USE_HW_BP | \
2319                               KVM_GUESTDBG_ENABLE)
2320
2321 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2322                                         struct kvm_guest_debug *dbg)
2323 {
2324         int rc = 0;
2325
2326         vcpu->guest_debug = 0;
2327         kvm_s390_clear_bp_data(vcpu);
2328
2329         if (dbg->control & ~VALID_GUESTDBG_FLAGS)
2330                 return -EINVAL;
2331         if (!sclp.has_gpere)
2332                 return -EINVAL;
2333
2334         if (dbg->control & KVM_GUESTDBG_ENABLE) {
2335                 vcpu->guest_debug = dbg->control;
2336                 /* enforce guest PER */
2337                 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2338
2339                 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2340                         rc = kvm_s390_import_bp_data(vcpu, dbg);
2341         } else {
2342                 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2343                 vcpu->arch.guestdbg.last_bp = 0;
2344         }
2345
2346         if (rc) {
2347                 vcpu->guest_debug = 0;
2348                 kvm_s390_clear_bp_data(vcpu);
2349                 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2350         }
2351
2352         return rc;
2353 }
2354
2355 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2356                                     struct kvm_mp_state *mp_state)
2357 {
2358         /* CHECK_STOP and LOAD are not supported yet */
2359         return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2360                                        KVM_MP_STATE_OPERATING;
2361 }
2362
2363 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2364                                     struct kvm_mp_state *mp_state)
2365 {
2366         int rc = 0;
2367
2368         /* user space knows about this interface - let it control the state */
2369         vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2370
2371         switch (mp_state->mp_state) {
2372         case KVM_MP_STATE_STOPPED:
2373                 kvm_s390_vcpu_stop(vcpu);
2374                 break;
2375         case KVM_MP_STATE_OPERATING:
2376                 kvm_s390_vcpu_start(vcpu);
2377                 break;
2378         case KVM_MP_STATE_LOAD:
2379         case KVM_MP_STATE_CHECK_STOP:
2380                 /* fall through - CHECK_STOP and LOAD are not supported yet */
2381         default:
2382                 rc = -ENXIO;
2383         }
2384
2385         return rc;
2386 }
2387
2388 static bool ibs_enabled(struct kvm_vcpu *vcpu)
2389 {
2390         return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2391 }
2392
2393 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2394 {
2395 retry:
2396         kvm_s390_vcpu_request_handled(vcpu);
2397         if (!vcpu->requests)
2398                 return 0;
2399         /*
2400          * We use MMU_RELOAD just to re-arm the ipte notifier for the
2401          * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
2402          * This ensures that the ipte instruction for this request has
2403          * already finished. We might race against a second unmapper that
2404          * wants to set the blocking bit. Lets just retry the request loop.
2405          */
2406         if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2407                 int rc;
2408                 rc = gmap_mprotect_notify(vcpu->arch.gmap,
2409                                           kvm_s390_get_prefix(vcpu),
2410                                           PAGE_SIZE * 2, PROT_WRITE);
2411                 if (rc) {
2412                         kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
2413                         return rc;
2414                 }
2415                 goto retry;
2416         }
2417
2418         if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2419                 vcpu->arch.sie_block->ihcpu = 0xffff;
2420                 goto retry;
2421         }
2422
2423         if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2424                 if (!ibs_enabled(vcpu)) {
2425                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
2426                         atomic_or(CPUSTAT_IBS,
2427                                         &vcpu->arch.sie_block->cpuflags);
2428                 }
2429                 goto retry;
2430         }
2431
2432         if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2433                 if (ibs_enabled(vcpu)) {
2434                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
2435                         atomic_andnot(CPUSTAT_IBS,
2436                                           &vcpu->arch.sie_block->cpuflags);
2437                 }
2438                 goto retry;
2439         }
2440
2441         if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
2442                 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2443                 goto retry;
2444         }
2445
2446         /* nothing to do, just clear the request */
2447         clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2448
2449         return 0;
2450 }
2451
2452 void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2453 {
2454         struct kvm_vcpu *vcpu;
2455         int i;
2456
2457         mutex_lock(&kvm->lock);
2458         preempt_disable();
2459         kvm->arch.epoch = tod - get_tod_clock();
2460         kvm_s390_vcpu_block_all(kvm);
2461         kvm_for_each_vcpu(i, vcpu, kvm)
2462                 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2463         kvm_s390_vcpu_unblock_all(kvm);
2464         preempt_enable();
2465         mutex_unlock(&kvm->lock);
2466 }
2467
2468 /**
2469  * kvm_arch_fault_in_page - fault-in guest page if necessary
2470  * @vcpu: The corresponding virtual cpu
2471  * @gpa: Guest physical address
2472  * @writable: Whether the page should be writable or not
2473  *
2474  * Make sure that a guest page has been faulted-in on the host.
2475  *
2476  * Return: Zero on success, negative error code otherwise.
2477  */
2478 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
2479 {
2480         return gmap_fault(vcpu->arch.gmap, gpa,
2481                           writable ? FAULT_FLAG_WRITE : 0);
2482 }
2483
2484 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2485                                       unsigned long token)
2486 {
2487         struct kvm_s390_interrupt inti;
2488         struct kvm_s390_irq irq;
2489
2490         if (start_token) {
2491                 irq.u.ext.ext_params2 = token;
2492                 irq.type = KVM_S390_INT_PFAULT_INIT;
2493                 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
2494         } else {
2495                 inti.type = KVM_S390_INT_PFAULT_DONE;
2496                 inti.parm64 = token;
2497                 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2498         }
2499 }
2500
2501 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2502                                      struct kvm_async_pf *work)
2503 {
2504         trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2505         __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2506 }
2507
2508 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2509                                  struct kvm_async_pf *work)
2510 {
2511         trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2512         __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2513 }
2514
2515 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2516                                struct kvm_async_pf *work)
2517 {
2518         /* s390 will always inject the page directly */
2519 }
2520
2521 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2522 {
2523         /*
2524          * s390 will always inject the page directly,
2525          * but we still want check_async_completion to cleanup
2526          */
2527         return true;
2528 }
2529
2530 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2531 {
2532         hva_t hva;
2533         struct kvm_arch_async_pf arch;
2534         int rc;
2535
2536         if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2537                 return 0;
2538         if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2539             vcpu->arch.pfault_compare)
2540                 return 0;
2541         if (psw_extint_disabled(vcpu))
2542                 return 0;
2543         if (kvm_s390_vcpu_has_irq(vcpu, 0))
2544                 return 0;
2545         if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2546                 return 0;
2547         if (!vcpu->arch.gmap->pfault_enabled)
2548                 return 0;
2549
2550         hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2551         hva += current->thread.gmap_addr & ~PAGE_MASK;
2552         if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
2553                 return 0;
2554
2555         rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2556         return rc;
2557 }
2558
2559 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
2560 {
2561         int rc, cpuflags;
2562
2563         /*
2564          * On s390 notifications for arriving pages will be delivered directly
2565          * to the guest but the house keeping for completed pfaults is
2566          * handled outside the worker.
2567          */
2568         kvm_check_async_pf_completion(vcpu);
2569
2570         vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2571         vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
2572
2573         if (need_resched())
2574                 schedule();
2575
2576         if (test_cpu_flag(CIF_MCCK_PENDING))
2577                 s390_handle_mcck();
2578
2579         if (!kvm_is_ucontrol(vcpu->kvm)) {
2580                 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2581                 if (rc)
2582                         return rc;
2583         }
2584
2585         rc = kvm_s390_handle_requests(vcpu);
2586         if (rc)
2587                 return rc;
2588
2589         if (guestdbg_enabled(vcpu)) {
2590                 kvm_s390_backup_guest_per_regs(vcpu);
2591                 kvm_s390_patch_guest_per_regs(vcpu);
2592         }
2593
2594         vcpu->arch.sie_block->icptcode = 0;
2595         cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2596         VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2597         trace_kvm_s390_sie_enter(vcpu, cpuflags);
2598
2599         return 0;
2600 }
2601
2602 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2603 {
2604         struct kvm_s390_pgm_info pgm_info = {
2605                 .code = PGM_ADDRESSING,
2606         };
2607         u8 opcode, ilen;
2608         int rc;
2609
2610         VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2611         trace_kvm_s390_sie_fault(vcpu);
2612
2613         /*
2614          * We want to inject an addressing exception, which is defined as a
2615          * suppressing or terminating exception. However, since we came here
2616          * by a DAT access exception, the PSW still points to the faulting
2617          * instruction since DAT exceptions are nullifying. So we've got
2618          * to look up the current opcode to get the length of the instruction
2619          * to be able to forward the PSW.
2620          */
2621         rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
2622         ilen = insn_length(opcode);
2623         if (rc < 0) {
2624                 return rc;
2625         } else if (rc) {
2626                 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2627                  * Forward by arbitrary ilc, injection will take care of
2628                  * nullification if necessary.
2629                  */
2630                 pgm_info = vcpu->arch.pgm;
2631                 ilen = 4;
2632         }
2633         pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
2634         kvm_s390_forward_psw(vcpu, ilen);
2635         return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
2636 }
2637
2638 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2639 {
2640         VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2641                    vcpu->arch.sie_block->icptcode);
2642         trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2643
2644         if (guestdbg_enabled(vcpu))
2645                 kvm_s390_restore_guest_per_regs(vcpu);
2646
2647         vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2648         vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
2649
2650         if (vcpu->arch.sie_block->icptcode > 0) {
2651                 int rc = kvm_handle_sie_intercept(vcpu);
2652
2653                 if (rc != -EOPNOTSUPP)
2654                         return rc;
2655                 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2656                 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2657                 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2658                 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2659                 return -EREMOTE;
2660         } else if (exit_reason != -EFAULT) {
2661                 vcpu->stat.exit_null++;
2662                 return 0;
2663         } else if (kvm_is_ucontrol(vcpu->kvm)) {
2664                 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2665                 vcpu->run->s390_ucontrol.trans_exc_code =
2666                                                 current->thread.gmap_addr;
2667                 vcpu->run->s390_ucontrol.pgm_code = 0x10;
2668                 return -EREMOTE;
2669         } else if (current->thread.gmap_pfault) {
2670                 trace_kvm_s390_major_guest_pfault(vcpu);
2671                 current->thread.gmap_pfault = 0;
2672                 if (kvm_arch_setup_async_pf(vcpu))
2673                         return 0;
2674                 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
2675         }
2676         return vcpu_post_run_fault_in_sie(vcpu);
2677 }
2678
2679 static int __vcpu_run(struct kvm_vcpu *vcpu)
2680 {
2681         int rc, exit_reason;
2682
2683         /*
2684          * We try to hold kvm->srcu during most of vcpu_run (except when run-
2685          * ning the guest), so that memslots (and other stuff) are protected
2686          */
2687         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2688
2689         do {
2690                 rc = vcpu_pre_run(vcpu);
2691                 if (rc)
2692                         break;
2693
2694                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2695                 /*
2696                  * As PF_VCPU will be used in fault handler, between
2697                  * guest_enter and guest_exit should be no uaccess.
2698                  */
2699                 local_irq_disable();
2700                 guest_enter_irqoff();
2701                 __disable_cpu_timer_accounting(vcpu);
2702                 local_irq_enable();
2703                 exit_reason = sie64a(vcpu->arch.sie_block,
2704                                      vcpu->run->s.regs.gprs);
2705                 local_irq_disable();
2706                 __enable_cpu_timer_accounting(vcpu);
2707                 guest_exit_irqoff();
2708                 local_irq_enable();
2709                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2710
2711                 rc = vcpu_post_run(vcpu, exit_reason);
2712         } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
2713
2714         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2715         return rc;
2716 }
2717
2718 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2719 {
2720         vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2721         vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2722         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2723                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2724         if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2725                 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
2726                 /* some control register changes require a tlb flush */
2727                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2728         }
2729         if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2730                 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
2731                 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2732                 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2733                 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2734                 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2735         }
2736         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2737                 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2738                 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2739                 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
2740                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2741                         kvm_clear_async_pf_completion_queue(vcpu);
2742         }
2743         /*
2744          * If userspace sets the riccb (e.g. after migration) to a valid state,
2745          * we should enable RI here instead of doing the lazy enablement.
2746          */
2747         if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
2748             test_kvm_facility(vcpu->kvm, 64)) {
2749                 struct runtime_instr_cb *riccb =
2750                         (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
2751
2752                 if (riccb->valid)
2753                         vcpu->arch.sie_block->ecb3 |= 0x01;
2754         }
2755         save_access_regs(vcpu->arch.host_acrs);
2756         restore_access_regs(vcpu->run->s.regs.acrs);
2757         /* save host (userspace) fprs/vrs */
2758         save_fpu_regs();
2759         vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
2760         vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
2761         if (MACHINE_HAS_VX)
2762                 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
2763         else
2764                 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
2765         current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
2766         if (test_fp_ctl(current->thread.fpu.fpc))
2767                 /* User space provided an invalid FPC, let's clear it */
2768                 current->thread.fpu.fpc = 0;
2769
2770         kvm_run->kvm_dirty_regs = 0;
2771 }
2772
2773 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2774 {
2775         kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2776         kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2777         kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2778         memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2779         kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
2780         kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2781         kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2782         kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2783         kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2784         kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2785         kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2786         kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2787         save_access_regs(vcpu->run->s.regs.acrs);
2788         restore_access_regs(vcpu->arch.host_acrs);
2789         /* Save guest register state */
2790         save_fpu_regs();
2791         vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
2792         /* Restore will be done lazily at return */
2793         current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
2794         current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
2795
2796 }
2797
2798 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2799 {
2800         int rc;
2801         sigset_t sigsaved;
2802
2803         if (kvm_run->immediate_exit)
2804                 return -EINTR;
2805
2806         if (guestdbg_exit_pending(vcpu)) {
2807                 kvm_s390_prepare_debug_exit(vcpu);
2808                 return 0;
2809         }
2810
2811         if (vcpu->sigset_active)
2812                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2813
2814         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2815                 kvm_s390_vcpu_start(vcpu);
2816         } else if (is_vcpu_stopped(vcpu)) {
2817                 pr_err_ratelimited("can't run stopped vcpu %d\n",
2818                                    vcpu->vcpu_id);
2819                 return -EINVAL;
2820         }
2821
2822         sync_regs(vcpu, kvm_run);
2823         enable_cpu_timer_accounting(vcpu);
2824
2825         might_fault();
2826         rc = __vcpu_run(vcpu);
2827
2828         if (signal_pending(current) && !rc) {
2829                 kvm_run->exit_reason = KVM_EXIT_INTR;
2830                 rc = -EINTR;
2831         }
2832
2833         if (guestdbg_exit_pending(vcpu) && !rc)  {
2834                 kvm_s390_prepare_debug_exit(vcpu);
2835                 rc = 0;
2836         }
2837
2838         if (rc == -EREMOTE) {
2839                 /* userspace support is needed, kvm_run has been prepared */
2840                 rc = 0;
2841         }
2842
2843         disable_cpu_timer_accounting(vcpu);
2844         store_regs(vcpu, kvm_run);
2845
2846         if (vcpu->sigset_active)
2847                 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2848
2849         vcpu->stat.exit_userspace++;
2850         return rc;
2851 }
2852
2853 /*
2854  * store status at address
2855  * we use have two special cases:
2856  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2857  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2858  */
2859 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
2860 {
2861         unsigned char archmode = 1;
2862         freg_t fprs[NUM_FPRS];
2863         unsigned int px;
2864         u64 clkcomp, cputm;
2865         int rc;
2866
2867         px = kvm_s390_get_prefix(vcpu);
2868         if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2869                 if (write_guest_abs(vcpu, 163, &archmode, 1))
2870                         return -EFAULT;
2871                 gpa = 0;
2872         } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2873                 if (write_guest_real(vcpu, 163, &archmode, 1))
2874                         return -EFAULT;
2875                 gpa = px;
2876         } else
2877                 gpa -= __LC_FPREGS_SAVE_AREA;
2878
2879         /* manually convert vector registers if necessary */
2880         if (MACHINE_HAS_VX) {
2881                 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
2882                 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2883                                      fprs, 128);
2884         } else {
2885                 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2886                                      vcpu->run->s.regs.fprs, 128);
2887         }
2888         rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
2889                               vcpu->run->s.regs.gprs, 128);
2890         rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
2891                               &vcpu->arch.sie_block->gpsw, 16);
2892         rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
2893                               &px, 4);
2894         rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
2895                               &vcpu->run->s.regs.fpc, 4);
2896         rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
2897                               &vcpu->arch.sie_block->todpr, 4);
2898         cputm = kvm_s390_get_cpu_timer(vcpu);
2899         rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
2900                               &cputm, 8);
2901         clkcomp = vcpu->arch.sie_block->ckc >> 8;
2902         rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
2903                               &clkcomp, 8);
2904         rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
2905                               &vcpu->run->s.regs.acrs, 64);
2906         rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
2907                               &vcpu->arch.sie_block->gcr, 128);
2908         return rc ? -EFAULT : 0;
2909 }
2910
2911 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2912 {
2913         /*
2914          * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2915          * switch in the run ioctl. Let's update our copies before we save
2916          * it into the save area
2917          */
2918         save_fpu_regs();
2919         vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
2920         save_access_regs(vcpu->run->s.regs.acrs);
2921
2922         return kvm_s390_store_status_unloaded(vcpu, addr);
2923 }
2924
2925 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2926 {
2927         kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2928         kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
2929 }
2930
2931 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2932 {
2933         unsigned int i;
2934         struct kvm_vcpu *vcpu;
2935
2936         kvm_for_each_vcpu(i, vcpu, kvm) {
2937                 __disable_ibs_on_vcpu(vcpu);
2938         }
2939 }
2940
2941 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2942 {
2943         if (!sclp.has_ibs)
2944                 return;
2945         kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2946         kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
2947 }
2948
2949 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2950 {
2951         int i, online_vcpus, started_vcpus = 0;
2952
2953         if (!is_vcpu_stopped(vcpu))
2954                 return;
2955
2956         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2957         /* Only one cpu at a time may enter/leave the STOPPED state. */
2958         spin_lock(&vcpu->kvm->arch.start_stop_lock);
2959         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2960
2961         for (i = 0; i < online_vcpus; i++) {
2962                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2963                         started_vcpus++;
2964         }
2965
2966         if (started_vcpus == 0) {
2967                 /* we're the only active VCPU -> speed it up */
2968                 __enable_ibs_on_vcpu(vcpu);
2969         } else if (started_vcpus == 1) {
2970                 /*
2971                  * As we are starting a second VCPU, we have to disable
2972                  * the IBS facility on all VCPUs to remove potentially
2973                  * oustanding ENABLE requests.
2974                  */
2975                 __disable_ibs_on_all_vcpus(vcpu->kvm);
2976         }
2977
2978         atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2979         /*
2980          * Another VCPU might have used IBS while we were offline.
2981          * Let's play safe and flush the VCPU at startup.
2982          */
2983         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2984         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2985         return;
2986 }
2987
2988 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2989 {
2990         int i, online_vcpus, started_vcpus = 0;
2991         struct kvm_vcpu *started_vcpu = NULL;
2992
2993         if (is_vcpu_stopped(vcpu))
2994                 return;
2995
2996         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2997         /* Only one cpu at a time may enter/leave the STOPPED state. */
2998         spin_lock(&vcpu->kvm->arch.start_stop_lock);
2999         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3000
3001         /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
3002         kvm_s390_clear_stop_irq(vcpu);
3003
3004         atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
3005         __disable_ibs_on_vcpu(vcpu);
3006
3007         for (i = 0; i < online_vcpus; i++) {
3008                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
3009                         started_vcpus++;
3010                         started_vcpu = vcpu->kvm->vcpus[i];
3011                 }
3012         }
3013
3014         if (started_vcpus == 1) {
3015                 /*
3016                  * As we only have one VCPU left, we want to enable the
3017                  * IBS facility for that VCPU to speed it up.
3018                  */
3019                 __enable_ibs_on_vcpu(started_vcpu);
3020         }
3021
3022         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
3023         return;
3024 }
3025
3026 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
3027                                      struct kvm_enable_cap *cap)
3028 {
3029         int r;
3030
3031         if (cap->flags)
3032                 return -EINVAL;
3033
3034         switch (cap->cap) {
3035         case KVM_CAP_S390_CSS_SUPPORT:
3036                 if (!vcpu->kvm->arch.css_support) {
3037                         vcpu->kvm->arch.css_support = 1;
3038                         VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
3039                         trace_kvm_s390_enable_css(vcpu->kvm);
3040                 }
3041                 r = 0;
3042                 break;
3043         default:
3044                 r = -EINVAL;
3045                 break;
3046         }
3047         return r;
3048 }
3049
3050 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
3051                                   struct kvm_s390_mem_op *mop)
3052 {
3053         void __user *uaddr = (void __user *)mop->buf;
3054         void *tmpbuf = NULL;
3055         int r, srcu_idx;
3056         const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
3057                                     | KVM_S390_MEMOP_F_CHECK_ONLY;
3058
3059         if (mop->flags & ~supported_flags)
3060                 return -EINVAL;
3061
3062         if (mop->size > MEM_OP_MAX_SIZE)
3063                 return -E2BIG;
3064
3065         if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
3066                 tmpbuf = vmalloc(mop->size);
3067                 if (!tmpbuf)
3068                         return -ENOMEM;
3069         }
3070
3071         srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3072
3073         switch (mop->op) {
3074         case KVM_S390_MEMOP_LOGICAL_READ:
3075                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
3076                         r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3077                                             mop->size, GACC_FETCH);
3078                         break;
3079                 }
3080                 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3081                 if (r == 0) {
3082                         if (copy_to_user(uaddr, tmpbuf, mop->size))
3083                                 r = -EFAULT;
3084                 }
3085                 break;
3086         case KVM_S390_MEMOP_LOGICAL_WRITE:
3087                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
3088                         r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3089                                             mop->size, GACC_STORE);
3090                         break;
3091                 }
3092                 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
3093                         r = -EFAULT;
3094                         break;
3095                 }
3096                 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3097                 break;
3098         default:
3099                 r = -EINVAL;
3100         }
3101
3102         srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
3103
3104         if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
3105                 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
3106
3107         vfree(tmpbuf);
3108         return r;
3109 }
3110
3111 long kvm_arch_vcpu_ioctl(struct file *filp,
3112                          unsigned int ioctl, unsigned long arg)
3113 {
3114         struct kvm_vcpu *vcpu = filp->private_data;
3115         void __user *argp = (void __user *)arg;
3116         int idx;
3117         long r;
3118
3119         switch (ioctl) {
3120         case KVM_S390_IRQ: {
3121                 struct kvm_s390_irq s390irq;
3122
3123                 r = -EFAULT;
3124                 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
3125                         break;
3126                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3127                 break;
3128         }
3129         case KVM_S390_INTERRUPT: {
3130                 struct kvm_s390_interrupt s390int;
3131                 struct kvm_s390_irq s390irq;
3132
3133                 r = -EFAULT;
3134                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
3135                         break;
3136                 if (s390int_to_s390irq(&s390int, &s390irq))
3137                         return -EINVAL;
3138                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3139                 break;
3140         }
3141         case KVM_S390_STORE_STATUS:
3142                 idx = srcu_read_lock(&vcpu->kvm->srcu);
3143                 r = kvm_s390_vcpu_store_status(vcpu, arg);
3144                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
3145                 break;
3146         case KVM_S390_SET_INITIAL_PSW: {
3147                 psw_t psw;
3148
3149                 r = -EFAULT;
3150                 if (copy_from_user(&psw, argp, sizeof(psw)))
3151                         break;
3152                 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3153                 break;
3154         }
3155         case KVM_S390_INITIAL_RESET:
3156                 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3157                 break;
3158         case KVM_SET_ONE_REG:
3159         case KVM_GET_ONE_REG: {
3160                 struct kvm_one_reg reg;
3161                 r = -EFAULT;
3162                 if (copy_from_user(&reg, argp, sizeof(reg)))
3163                         break;
3164                 if (ioctl == KVM_SET_ONE_REG)
3165                         r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
3166                 else
3167                         r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
3168                 break;
3169         }
3170 #ifdef CONFIG_KVM_S390_UCONTROL
3171         case KVM_S390_UCAS_MAP: {
3172                 struct kvm_s390_ucas_mapping ucasmap;
3173
3174                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3175                         r = -EFAULT;
3176                         break;
3177                 }
3178
3179                 if (!kvm_is_ucontrol(vcpu->kvm)) {
3180                         r = -EINVAL;
3181                         break;
3182                 }
3183
3184                 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3185                                      ucasmap.vcpu_addr, ucasmap.length);
3186                 break;
3187         }
3188         case KVM_S390_UCAS_UNMAP: {
3189                 struct kvm_s390_ucas_mapping ucasmap;
3190
3191                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3192                         r = -EFAULT;
3193                         break;
3194                 }
3195
3196                 if (!kvm_is_ucontrol(vcpu->kvm)) {
3197                         r = -EINVAL;
3198                         break;
3199                 }
3200
3201                 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3202                         ucasmap.length);
3203                 break;
3204         }
3205 #endif
3206         case KVM_S390_VCPU_FAULT: {
3207                 r = gmap_fault(vcpu->arch.gmap, arg, 0);
3208                 break;
3209         }
3210         case KVM_ENABLE_CAP:
3211         {
3212                 struct kvm_enable_cap cap;
3213                 r = -EFAULT;
3214                 if (copy_from_user(&cap, argp, sizeof(cap)))
3215                         break;
3216                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3217                 break;
3218         }
3219         case KVM_S390_MEM_OP: {
3220                 struct kvm_s390_mem_op mem_op;
3221
3222                 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3223                         r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3224                 else
3225                         r = -EFAULT;
3226                 break;
3227         }
3228         case KVM_S390_SET_IRQ_STATE: {
3229                 struct kvm_s390_irq_state irq_state;
3230
3231                 r = -EFAULT;
3232                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3233                         break;
3234                 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3235                     irq_state.len == 0 ||
3236                     irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3237                         r = -EINVAL;
3238                         break;
3239                 }
3240                 r = kvm_s390_set_irq_state(vcpu,
3241                                            (void __user *) irq_state.buf,
3242                                            irq_state.len);
3243                 break;
3244         }
3245         case KVM_S390_GET_IRQ_STATE: {
3246                 struct kvm_s390_irq_state irq_state;
3247
3248                 r = -EFAULT;
3249                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3250                         break;
3251                 if (irq_state.len == 0) {
3252                         r = -EINVAL;
3253                         break;
3254                 }
3255                 r = kvm_s390_get_irq_state(vcpu,
3256                                            (__u8 __user *)  irq_state.buf,
3257                                            irq_state.len);
3258                 break;
3259         }
3260         default:
3261                 r = -ENOTTY;
3262         }
3263         return r;
3264 }
3265
3266 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3267 {
3268 #ifdef CONFIG_KVM_S390_UCONTROL
3269         if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3270                  && (kvm_is_ucontrol(vcpu->kvm))) {
3271                 vmf->page = virt_to_page(vcpu->arch.sie_block);
3272                 get_page(vmf->page);
3273                 return 0;
3274         }
3275 #endif
3276         return VM_FAULT_SIGBUS;
3277 }
3278
3279 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
3280                             unsigned long npages)
3281 {
3282         return 0;
3283 }
3284
3285 /* Section: memory related */
3286 int kvm_arch_prepare_memory_region(struct kvm *kvm,
3287                                    struct kvm_memory_slot *memslot,
3288                                    const struct kvm_userspace_memory_region *mem,
3289                                    enum kvm_mr_change change)
3290 {
3291         /* A few sanity checks. We can have memory slots which have to be
3292            located/ended at a segment boundary (1MB). The memory in userland is
3293            ok to be fragmented into various different vmas. It is okay to mmap()
3294            and munmap() stuff in this slot after doing this call at any time */
3295
3296         if (mem->userspace_addr & 0xffffful)
3297                 return -EINVAL;
3298
3299         if (mem->memory_size & 0xffffful)
3300                 return -EINVAL;
3301
3302         if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
3303                 return -EINVAL;
3304
3305         return 0;
3306 }
3307
3308 void kvm_arch_commit_memory_region(struct kvm *kvm,
3309                                 const struct kvm_userspace_memory_region *mem,
3310                                 const struct kvm_memory_slot *old,
3311                                 const struct kvm_memory_slot *new,
3312                                 enum kvm_mr_change change)
3313 {
3314         int rc;
3315
3316         /* If the basics of the memslot do not change, we do not want
3317          * to update the gmap. Every update causes several unnecessary
3318          * segment translation exceptions. This is usually handled just
3319          * fine by the normal fault handler + gmap, but it will also
3320          * cause faults on the prefix page of running guest CPUs.
3321          */
3322         if (old->userspace_addr == mem->userspace_addr &&
3323             old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
3324             old->npages * PAGE_SIZE == mem->memory_size)
3325                 return;
3326
3327         rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
3328                 mem->guest_phys_addr, mem->memory_size);
3329         if (rc)
3330                 pr_warn("failed to commit memory region\n");
3331         return;
3332 }
3333
3334 static inline unsigned long nonhyp_mask(int i)
3335 {
3336         unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
3337
3338         return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
3339 }
3340
3341 void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
3342 {
3343         vcpu->valid_wakeup = false;
3344 }
3345
3346 static int __init kvm_s390_init(void)
3347 {
3348         int i;
3349
3350         if (!sclp.has_sief2) {
3351                 pr_info("SIE not available\n");
3352                 return -ENODEV;
3353         }
3354
3355         for (i = 0; i < 16; i++)
3356                 kvm_s390_fac_list_mask[i] |=
3357                         S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3358
3359         return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
3360 }
3361
3362 static void __exit kvm_s390_exit(void)
3363 {
3364         kvm_exit();
3365 }
3366
3367 module_init(kvm_s390_init);
3368 module_exit(kvm_s390_exit);
3369
3370 /*
3371  * Enable autoloading of the kvm module.
3372  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3373  * since x86 takes a different approach.
3374  */
3375 #include <linux/miscdevice.h>
3376 MODULE_ALIAS_MISCDEV(KVM_MINOR);
3377 MODULE_ALIAS("devname:kvm");