2 * hosting zSeries kernel virtual machines
4 * Copyright IBM Corp. 2008, 2009
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
16 #include <linux/compiler.h>
17 #include <linux/err.h>
19 #include <linux/hrtimer.h>
20 #include <linux/init.h>
21 #include <linux/kvm.h>
22 #include <linux/kvm_host.h>
23 #include <linux/module.h>
24 #include <linux/slab.h>
25 #include <linux/timer.h>
26 #include <asm/asm-offsets.h>
27 #include <asm/lowcore.h>
28 #include <asm/pgtable.h>
30 #include <asm/switch_to.h>
31 #include <asm/facility.h>
36 #define CREATE_TRACE_POINTS
38 #include "trace-s390.h"
40 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
42 struct kvm_stats_debugfs_item debugfs_entries[] = {
43 { "userspace_handled", VCPU_STAT(exit_userspace) },
44 { "exit_null", VCPU_STAT(exit_null) },
45 { "exit_validity", VCPU_STAT(exit_validity) },
46 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
47 { "exit_external_request", VCPU_STAT(exit_external_request) },
48 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
49 { "exit_instruction", VCPU_STAT(exit_instruction) },
50 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
51 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
52 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
53 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
54 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
55 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
56 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
57 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
58 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
59 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
60 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
61 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
62 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
63 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
64 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
65 { "instruction_spx", VCPU_STAT(instruction_spx) },
66 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
67 { "instruction_stap", VCPU_STAT(instruction_stap) },
68 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
69 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
70 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
71 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
72 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
73 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
74 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
75 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
76 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
77 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
78 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
79 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
80 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
81 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
82 { "diagnose_10", VCPU_STAT(diagnose_10) },
83 { "diagnose_44", VCPU_STAT(diagnose_44) },
84 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
88 unsigned long *vfacilities;
89 static struct gmap_notifier gmap_notifier;
91 /* test availability of vfacility */
92 static inline int test_vfacility(unsigned long nr)
94 return __test_facility(nr, (void *) vfacilities);
97 /* Section: not file related */
98 int kvm_arch_hardware_enable(void *garbage)
100 /* every s390 is virtualization enabled ;-) */
104 void kvm_arch_hardware_disable(void *garbage)
108 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
110 int kvm_arch_hardware_setup(void)
112 gmap_notifier.notifier_call = kvm_gmap_notifier;
113 gmap_register_ipte_notifier(&gmap_notifier);
117 void kvm_arch_hardware_unsetup(void)
119 gmap_unregister_ipte_notifier(&gmap_notifier);
122 void kvm_arch_check_processor_compat(void *rtn)
126 int kvm_arch_init(void *opaque)
131 void kvm_arch_exit(void)
135 /* Section: device related */
136 long kvm_arch_dev_ioctl(struct file *filp,
137 unsigned int ioctl, unsigned long arg)
139 if (ioctl == KVM_S390_ENABLE_SIE)
140 return s390_enable_sie();
144 int kvm_dev_ioctl_check_extension(long ext)
149 case KVM_CAP_S390_PSW:
150 case KVM_CAP_S390_GMAP:
151 case KVM_CAP_SYNC_MMU:
152 #ifdef CONFIG_KVM_S390_UCONTROL
153 case KVM_CAP_S390_UCONTROL:
155 case KVM_CAP_SYNC_REGS:
156 case KVM_CAP_ONE_REG:
157 case KVM_CAP_ENABLE_CAP:
158 case KVM_CAP_S390_CSS_SUPPORT:
159 case KVM_CAP_IOEVENTFD:
162 case KVM_CAP_NR_VCPUS:
163 case KVM_CAP_MAX_VCPUS:
166 case KVM_CAP_NR_MEMSLOTS:
167 r = KVM_USER_MEM_SLOTS;
169 case KVM_CAP_S390_COW:
170 r = MACHINE_HAS_ESOP;
178 /* Section: vm related */
180 * Get (and clear) the dirty memory log for a memory slot.
182 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
183 struct kvm_dirty_log *log)
188 long kvm_arch_vm_ioctl(struct file *filp,
189 unsigned int ioctl, unsigned long arg)
191 struct kvm *kvm = filp->private_data;
192 void __user *argp = (void __user *)arg;
196 case KVM_S390_INTERRUPT: {
197 struct kvm_s390_interrupt s390int;
200 if (copy_from_user(&s390int, argp, sizeof(s390int)))
202 r = kvm_s390_inject_vm(kvm, &s390int);
212 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
218 #ifdef CONFIG_KVM_S390_UCONTROL
219 if (type & ~KVM_VM_S390_UCONTROL)
221 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
228 rc = s390_enable_sie();
234 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
238 sprintf(debug_name, "kvm-%u", current->pid);
240 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
244 spin_lock_init(&kvm->arch.float_int.lock);
245 INIT_LIST_HEAD(&kvm->arch.float_int.list);
247 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
248 VM_EVENT(kvm, 3, "%s", "vm created");
250 if (type & KVM_VM_S390_UCONTROL) {
251 kvm->arch.gmap = NULL;
253 kvm->arch.gmap = gmap_alloc(current->mm);
256 kvm->arch.gmap->private = kvm;
259 kvm->arch.css_support = 0;
263 debug_unregister(kvm->arch.dbf);
265 free_page((unsigned long)(kvm->arch.sca));
270 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
272 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
273 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
274 if (!kvm_is_ucontrol(vcpu->kvm)) {
275 clear_bit(63 - vcpu->vcpu_id,
276 (unsigned long *) &vcpu->kvm->arch.sca->mcn);
277 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
278 (__u64) vcpu->arch.sie_block)
279 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
283 if (kvm_is_ucontrol(vcpu->kvm))
284 gmap_free(vcpu->arch.gmap);
286 free_page((unsigned long)(vcpu->arch.sie_block));
287 kvm_vcpu_uninit(vcpu);
288 kmem_cache_free(kvm_vcpu_cache, vcpu);
291 static void kvm_free_vcpus(struct kvm *kvm)
294 struct kvm_vcpu *vcpu;
296 kvm_for_each_vcpu(i, vcpu, kvm)
297 kvm_arch_vcpu_destroy(vcpu);
299 mutex_lock(&kvm->lock);
300 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
301 kvm->vcpus[i] = NULL;
303 atomic_set(&kvm->online_vcpus, 0);
304 mutex_unlock(&kvm->lock);
307 void kvm_arch_sync_events(struct kvm *kvm)
311 void kvm_arch_destroy_vm(struct kvm *kvm)
314 free_page((unsigned long)(kvm->arch.sca));
315 debug_unregister(kvm->arch.dbf);
316 if (!kvm_is_ucontrol(kvm))
317 gmap_free(kvm->arch.gmap);
320 /* Section: vcpu related */
321 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
323 if (kvm_is_ucontrol(vcpu->kvm)) {
324 vcpu->arch.gmap = gmap_alloc(current->mm);
325 if (!vcpu->arch.gmap)
327 vcpu->arch.gmap->private = vcpu->kvm;
331 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
332 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
339 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
344 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
346 save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
347 save_fp_regs(vcpu->arch.host_fpregs.fprs);
348 save_access_regs(vcpu->arch.host_acrs);
349 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
350 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
351 restore_access_regs(vcpu->run->s.regs.acrs);
352 gmap_enable(vcpu->arch.gmap);
353 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
356 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
358 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
359 gmap_disable(vcpu->arch.gmap);
360 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
361 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
362 save_access_regs(vcpu->run->s.regs.acrs);
363 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
364 restore_fp_regs(vcpu->arch.host_fpregs.fprs);
365 restore_access_regs(vcpu->arch.host_acrs);
368 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
370 /* this equals initial cpu reset in pop, but we don't switch to ESA */
371 vcpu->arch.sie_block->gpsw.mask = 0UL;
372 vcpu->arch.sie_block->gpsw.addr = 0UL;
373 kvm_s390_set_prefix(vcpu, 0);
374 vcpu->arch.sie_block->cputm = 0UL;
375 vcpu->arch.sie_block->ckc = 0UL;
376 vcpu->arch.sie_block->todpr = 0;
377 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
378 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
379 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
380 vcpu->arch.guest_fpregs.fpc = 0;
381 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
382 vcpu->arch.sie_block->gbea = 1;
383 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
386 int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
391 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
393 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
397 vcpu->arch.sie_block->ecb = 6;
398 vcpu->arch.sie_block->ecb2 = 8;
399 vcpu->arch.sie_block->eca = 0xC1002001U;
400 vcpu->arch.sie_block->fac = (int) (long) vfacilities;
401 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
402 tasklet_init(&vcpu->arch.tasklet, kvm_s390_tasklet,
403 (unsigned long) vcpu);
404 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
405 get_cpu_id(&vcpu->arch.cpu_id);
406 vcpu->arch.cpu_id.version = 0xff;
410 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
413 struct kvm_vcpu *vcpu;
416 if (id >= KVM_MAX_VCPUS)
421 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
425 vcpu->arch.sie_block = (struct kvm_s390_sie_block *)
426 get_zeroed_page(GFP_KERNEL);
428 if (!vcpu->arch.sie_block)
431 vcpu->arch.sie_block->icpua = id;
432 if (!kvm_is_ucontrol(kvm)) {
433 if (!kvm->arch.sca) {
437 if (!kvm->arch.sca->cpu[id].sda)
438 kvm->arch.sca->cpu[id].sda =
439 (__u64) vcpu->arch.sie_block;
440 vcpu->arch.sie_block->scaoh =
441 (__u32)(((__u64)kvm->arch.sca) >> 32);
442 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
443 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
446 spin_lock_init(&vcpu->arch.local_int.lock);
447 INIT_LIST_HEAD(&vcpu->arch.local_int.list);
448 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
449 spin_lock(&kvm->arch.float_int.lock);
450 kvm->arch.float_int.local_int[id] = &vcpu->arch.local_int;
451 vcpu->arch.local_int.wq = &vcpu->wq;
452 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
453 spin_unlock(&kvm->arch.float_int.lock);
455 rc = kvm_vcpu_init(vcpu, kvm, id);
457 goto out_free_sie_block;
458 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
459 vcpu->arch.sie_block);
460 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
464 free_page((unsigned long)(vcpu->arch.sie_block));
466 kmem_cache_free(kvm_vcpu_cache, vcpu);
471 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
473 /* kvm common code refers to this, but never calls it */
478 void s390_vcpu_block(struct kvm_vcpu *vcpu)
480 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
483 void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
485 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
489 * Kick a guest cpu out of SIE and wait until SIE is not running.
490 * If the CPU is not running (e.g. waiting as idle) the function will
491 * return immediately. */
492 void exit_sie(struct kvm_vcpu *vcpu)
494 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
495 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
499 /* Kick a guest cpu out of SIE and prevent SIE-reentry */
500 void exit_sie_sync(struct kvm_vcpu *vcpu)
502 s390_vcpu_block(vcpu);
506 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
509 struct kvm *kvm = gmap->private;
510 struct kvm_vcpu *vcpu;
512 kvm_for_each_vcpu(i, vcpu, kvm) {
513 /* match against both prefix pages */
514 if (vcpu->arch.sie_block->prefix == (address & ~0x1000UL)) {
515 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
516 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
522 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
524 /* kvm common code refers to this, but never calls it */
529 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
530 struct kvm_one_reg *reg)
535 case KVM_REG_S390_TODPR:
536 r = put_user(vcpu->arch.sie_block->todpr,
537 (u32 __user *)reg->addr);
539 case KVM_REG_S390_EPOCHDIFF:
540 r = put_user(vcpu->arch.sie_block->epoch,
541 (u64 __user *)reg->addr);
543 case KVM_REG_S390_CPU_TIMER:
544 r = put_user(vcpu->arch.sie_block->cputm,
545 (u64 __user *)reg->addr);
547 case KVM_REG_S390_CLOCK_COMP:
548 r = put_user(vcpu->arch.sie_block->ckc,
549 (u64 __user *)reg->addr);
558 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
559 struct kvm_one_reg *reg)
564 case KVM_REG_S390_TODPR:
565 r = get_user(vcpu->arch.sie_block->todpr,
566 (u32 __user *)reg->addr);
568 case KVM_REG_S390_EPOCHDIFF:
569 r = get_user(vcpu->arch.sie_block->epoch,
570 (u64 __user *)reg->addr);
572 case KVM_REG_S390_CPU_TIMER:
573 r = get_user(vcpu->arch.sie_block->cputm,
574 (u64 __user *)reg->addr);
576 case KVM_REG_S390_CLOCK_COMP:
577 r = get_user(vcpu->arch.sie_block->ckc,
578 (u64 __user *)reg->addr);
587 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
589 kvm_s390_vcpu_initial_reset(vcpu);
593 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
595 memcpy(&vcpu->run->s.regs.gprs, ®s->gprs, sizeof(regs->gprs));
599 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
601 memcpy(®s->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
605 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
606 struct kvm_sregs *sregs)
608 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
609 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
610 restore_access_regs(vcpu->run->s.regs.acrs);
614 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
615 struct kvm_sregs *sregs)
617 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
618 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
622 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
624 if (test_fp_ctl(fpu->fpc))
626 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
627 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
628 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
629 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
633 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
635 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
636 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
640 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
644 if (!(atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_STOPPED))
647 vcpu->run->psw_mask = psw.mask;
648 vcpu->run->psw_addr = psw.addr;
653 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
654 struct kvm_translation *tr)
656 return -EINVAL; /* not implemented yet */
659 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
660 struct kvm_guest_debug *dbg)
662 return -EINVAL; /* not implemented yet */
665 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
666 struct kvm_mp_state *mp_state)
668 return -EINVAL; /* not implemented yet */
671 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
672 struct kvm_mp_state *mp_state)
674 return -EINVAL; /* not implemented yet */
677 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
680 * We use MMU_RELOAD just to re-arm the ipte notifier for the
681 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
682 * This ensures that the ipte instruction for this request has
683 * already finished. We might race against a second unmapper that
684 * wants to set the blocking bit. Lets just retry the request loop.
686 while (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
688 rc = gmap_ipte_notify(vcpu->arch.gmap,
689 vcpu->arch.sie_block->prefix,
693 s390_vcpu_unblock(vcpu);
698 static int __vcpu_run(struct kvm_vcpu *vcpu)
702 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
707 if (test_thread_flag(TIF_MCCK_PENDING))
710 if (!kvm_is_ucontrol(vcpu->kvm))
711 kvm_s390_deliver_pending_interrupts(vcpu);
713 rc = kvm_s390_handle_requests(vcpu);
717 vcpu->arch.sie_block->icptcode = 0;
718 VCPU_EVENT(vcpu, 6, "entering sie flags %x",
719 atomic_read(&vcpu->arch.sie_block->cpuflags));
720 trace_kvm_s390_sie_enter(vcpu,
721 atomic_read(&vcpu->arch.sie_block->cpuflags));
724 * As PF_VCPU will be used in fault handler, between guest_enter
725 * and guest_exit should be no uaccess.
730 rc = sie64a(vcpu->arch.sie_block, vcpu->run->s.regs.gprs);
733 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
734 vcpu->arch.sie_block->icptcode);
735 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
740 if (kvm_is_ucontrol(vcpu->kvm)) {
741 rc = SIE_INTERCEPT_UCONTROL;
743 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
744 trace_kvm_s390_sie_fault(vcpu);
745 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
749 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
753 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
759 if (vcpu->sigset_active)
760 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
762 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
764 BUG_ON(vcpu->kvm->arch.float_int.local_int[vcpu->vcpu_id] == NULL);
766 switch (kvm_run->exit_reason) {
767 case KVM_EXIT_S390_SIEIC:
768 case KVM_EXIT_UNKNOWN:
770 case KVM_EXIT_S390_RESET:
771 case KVM_EXIT_S390_UCONTROL:
772 case KVM_EXIT_S390_TSCH:
778 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
779 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
780 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX) {
781 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_PREFIX;
782 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
784 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
785 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_CRS;
786 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
787 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
793 rc = __vcpu_run(vcpu);
796 if (kvm_is_ucontrol(vcpu->kvm))
799 rc = kvm_handle_sie_intercept(vcpu);
800 } while (!signal_pending(current) && !rc);
802 if (rc == SIE_INTERCEPT_RERUNVCPU)
805 if (signal_pending(current) && !rc) {
806 kvm_run->exit_reason = KVM_EXIT_INTR;
810 #ifdef CONFIG_KVM_S390_UCONTROL
811 if (rc == SIE_INTERCEPT_UCONTROL) {
812 kvm_run->exit_reason = KVM_EXIT_S390_UCONTROL;
813 kvm_run->s390_ucontrol.trans_exc_code =
814 current->thread.gmap_addr;
815 kvm_run->s390_ucontrol.pgm_code = 0x10;
820 if (rc == -EOPNOTSUPP) {
821 /* intercept cannot be handled in-kernel, prepare kvm-run */
822 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC;
823 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
824 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
825 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
829 if (rc == -EREMOTE) {
830 /* intercept was handled, but userspace support is needed
831 * kvm_run has been prepared by the handler */
835 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
836 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
837 kvm_run->s.regs.prefix = vcpu->arch.sie_block->prefix;
838 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
840 if (vcpu->sigset_active)
841 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
843 vcpu->stat.exit_userspace++;
847 static int __guestcopy(struct kvm_vcpu *vcpu, u64 guestdest, void *from,
848 unsigned long n, int prefix)
851 return copy_to_guest(vcpu, guestdest, from, n);
853 return copy_to_guest_absolute(vcpu, guestdest, from, n);
857 * store status at address
858 * we use have two special cases:
859 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
860 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
862 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
864 unsigned char archmode = 1;
867 if (addr == KVM_S390_STORE_STATUS_NOADDR) {
868 if (copy_to_guest_absolute(vcpu, 163ul, &archmode, 1))
870 addr = SAVE_AREA_BASE;
872 } else if (addr == KVM_S390_STORE_STATUS_PREFIXED) {
873 if (copy_to_guest(vcpu, 163ul, &archmode, 1))
875 addr = SAVE_AREA_BASE;
881 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
882 * copying in vcpu load/put. Lets update our copies before we save
883 * it into the save area
885 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
886 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
887 save_access_regs(vcpu->run->s.regs.acrs);
889 if (__guestcopy(vcpu, addr + offsetof(struct save_area, fp_regs),
890 vcpu->arch.guest_fpregs.fprs, 128, prefix))
893 if (__guestcopy(vcpu, addr + offsetof(struct save_area, gp_regs),
894 vcpu->run->s.regs.gprs, 128, prefix))
897 if (__guestcopy(vcpu, addr + offsetof(struct save_area, psw),
898 &vcpu->arch.sie_block->gpsw, 16, prefix))
901 if (__guestcopy(vcpu, addr + offsetof(struct save_area, pref_reg),
902 &vcpu->arch.sie_block->prefix, 4, prefix))
905 if (__guestcopy(vcpu,
906 addr + offsetof(struct save_area, fp_ctrl_reg),
907 &vcpu->arch.guest_fpregs.fpc, 4, prefix))
910 if (__guestcopy(vcpu, addr + offsetof(struct save_area, tod_reg),
911 &vcpu->arch.sie_block->todpr, 4, prefix))
914 if (__guestcopy(vcpu, addr + offsetof(struct save_area, timer),
915 &vcpu->arch.sie_block->cputm, 8, prefix))
918 if (__guestcopy(vcpu, addr + offsetof(struct save_area, clk_cmp),
919 &vcpu->arch.sie_block->ckc, 8, prefix))
922 if (__guestcopy(vcpu, addr + offsetof(struct save_area, acc_regs),
923 &vcpu->run->s.regs.acrs, 64, prefix))
926 if (__guestcopy(vcpu,
927 addr + offsetof(struct save_area, ctrl_regs),
928 &vcpu->arch.sie_block->gcr, 128, prefix))
933 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
934 struct kvm_enable_cap *cap)
942 case KVM_CAP_S390_CSS_SUPPORT:
943 if (!vcpu->kvm->arch.css_support) {
944 vcpu->kvm->arch.css_support = 1;
945 trace_kvm_s390_enable_css(vcpu->kvm);
956 long kvm_arch_vcpu_ioctl(struct file *filp,
957 unsigned int ioctl, unsigned long arg)
959 struct kvm_vcpu *vcpu = filp->private_data;
960 void __user *argp = (void __user *)arg;
964 case KVM_S390_INTERRUPT: {
965 struct kvm_s390_interrupt s390int;
968 if (copy_from_user(&s390int, argp, sizeof(s390int)))
970 r = kvm_s390_inject_vcpu(vcpu, &s390int);
973 case KVM_S390_STORE_STATUS:
974 r = kvm_s390_vcpu_store_status(vcpu, arg);
976 case KVM_S390_SET_INITIAL_PSW: {
980 if (copy_from_user(&psw, argp, sizeof(psw)))
982 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
985 case KVM_S390_INITIAL_RESET:
986 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
988 case KVM_SET_ONE_REG:
989 case KVM_GET_ONE_REG: {
990 struct kvm_one_reg reg;
992 if (copy_from_user(®, argp, sizeof(reg)))
994 if (ioctl == KVM_SET_ONE_REG)
995 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, ®);
997 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, ®);
1000 #ifdef CONFIG_KVM_S390_UCONTROL
1001 case KVM_S390_UCAS_MAP: {
1002 struct kvm_s390_ucas_mapping ucasmap;
1004 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1009 if (!kvm_is_ucontrol(vcpu->kvm)) {
1014 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
1015 ucasmap.vcpu_addr, ucasmap.length);
1018 case KVM_S390_UCAS_UNMAP: {
1019 struct kvm_s390_ucas_mapping ucasmap;
1021 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1026 if (!kvm_is_ucontrol(vcpu->kvm)) {
1031 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
1036 case KVM_S390_VCPU_FAULT: {
1037 r = gmap_fault(arg, vcpu->arch.gmap);
1038 if (!IS_ERR_VALUE(r))
1042 case KVM_ENABLE_CAP:
1044 struct kvm_enable_cap cap;
1046 if (copy_from_user(&cap, argp, sizeof(cap)))
1048 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
1057 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
1059 #ifdef CONFIG_KVM_S390_UCONTROL
1060 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
1061 && (kvm_is_ucontrol(vcpu->kvm))) {
1062 vmf->page = virt_to_page(vcpu->arch.sie_block);
1063 get_page(vmf->page);
1067 return VM_FAULT_SIGBUS;
1070 void kvm_arch_free_memslot(struct kvm_memory_slot *free,
1071 struct kvm_memory_slot *dont)
1075 int kvm_arch_create_memslot(struct kvm_memory_slot *slot, unsigned long npages)
1080 void kvm_arch_memslots_updated(struct kvm *kvm)
1084 /* Section: memory related */
1085 int kvm_arch_prepare_memory_region(struct kvm *kvm,
1086 struct kvm_memory_slot *memslot,
1087 struct kvm_userspace_memory_region *mem,
1088 enum kvm_mr_change change)
1090 /* A few sanity checks. We can have memory slots which have to be
1091 located/ended at a segment boundary (1MB). The memory in userland is
1092 ok to be fragmented into various different vmas. It is okay to mmap()
1093 and munmap() stuff in this slot after doing this call at any time */
1095 if (mem->userspace_addr & 0xffffful)
1098 if (mem->memory_size & 0xffffful)
1104 void kvm_arch_commit_memory_region(struct kvm *kvm,
1105 struct kvm_userspace_memory_region *mem,
1106 const struct kvm_memory_slot *old,
1107 enum kvm_mr_change change)
1111 /* If the basics of the memslot do not change, we do not want
1112 * to update the gmap. Every update causes several unnecessary
1113 * segment translation exceptions. This is usually handled just
1114 * fine by the normal fault handler + gmap, but it will also
1115 * cause faults on the prefix page of running guest CPUs.
1117 if (old->userspace_addr == mem->userspace_addr &&
1118 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
1119 old->npages * PAGE_SIZE == mem->memory_size)
1122 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
1123 mem->guest_phys_addr, mem->memory_size);
1125 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
1129 void kvm_arch_flush_shadow_all(struct kvm *kvm)
1133 void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
1134 struct kvm_memory_slot *slot)
1138 static int __init kvm_s390_init(void)
1141 ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
1146 * guests can ask for up to 255+1 double words, we need a full page
1147 * to hold the maximum amount of facilities. On the other hand, we
1148 * only set facilities that are known to work in KVM.
1150 vfacilities = (unsigned long *) get_zeroed_page(GFP_KERNEL|GFP_DMA);
1155 memcpy(vfacilities, S390_lowcore.stfle_fac_list, 16);
1156 vfacilities[0] &= 0xff82fff3f47c0000UL;
1157 vfacilities[1] &= 0x001c000000000000UL;
1161 static void __exit kvm_s390_exit(void)
1163 free_page((unsigned long) vfacilities);
1167 module_init(kvm_s390_init);
1168 module_exit(kvm_s390_exit);
1171 * Enable autoloading of the kvm module.
1172 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
1173 * since x86 takes a different approach.
1175 #include <linux/miscdevice.h>
1176 MODULE_ALIAS_MISCDEV(KVM_MINOR);
1177 MODULE_ALIAS("devname:kvm");