2 * guest access functions
4 * Copyright IBM Corp. 2014
8 #include <linux/vmalloc.h>
9 #include <linux/mm_types.h>
10 #include <linux/err.h>
12 #include <asm/pgtable.h>
16 #include <asm/switch_to.h>
21 unsigned long origin : 52; /* Region- or Segment-Table Origin */
23 unsigned long g : 1; /* Subspace Group Control */
24 unsigned long p : 1; /* Private Space Control */
25 unsigned long s : 1; /* Storage-Alteration-Event Control */
26 unsigned long x : 1; /* Space-Switch-Event Control */
27 unsigned long r : 1; /* Real-Space Control */
29 unsigned long dt : 2; /* Designation-Type Control */
30 unsigned long tl : 2; /* Region- or Segment-Table Length */
35 ASCE_TYPE_SEGMENT = 0,
36 ASCE_TYPE_REGION3 = 1,
37 ASCE_TYPE_REGION2 = 2,
41 union region1_table_entry {
44 unsigned long rto: 52;/* Region-Table Origin */
46 unsigned long p : 1; /* DAT-Protection Bit */
48 unsigned long tf : 2; /* Region-Second-Table Offset */
49 unsigned long i : 1; /* Region-Invalid Bit */
51 unsigned long tt : 2; /* Table-Type Bits */
52 unsigned long tl : 2; /* Region-Second-Table Length */
56 union region2_table_entry {
59 unsigned long rto: 52;/* Region-Table Origin */
61 unsigned long p : 1; /* DAT-Protection Bit */
63 unsigned long tf : 2; /* Region-Third-Table Offset */
64 unsigned long i : 1; /* Region-Invalid Bit */
66 unsigned long tt : 2; /* Table-Type Bits */
67 unsigned long tl : 2; /* Region-Third-Table Length */
71 struct region3_table_entry_fc0 {
72 unsigned long sto: 52;/* Segment-Table Origin */
74 unsigned long fc : 1; /* Format-Control */
75 unsigned long p : 1; /* DAT-Protection Bit */
77 unsigned long tf : 2; /* Segment-Table Offset */
78 unsigned long i : 1; /* Region-Invalid Bit */
79 unsigned long cr : 1; /* Common-Region Bit */
80 unsigned long tt : 2; /* Table-Type Bits */
81 unsigned long tl : 2; /* Segment-Table Length */
84 struct region3_table_entry_fc1 {
85 unsigned long rfaa : 33; /* Region-Frame Absolute Address */
87 unsigned long av : 1; /* ACCF-Validity Control */
88 unsigned long acc: 4; /* Access-Control Bits */
89 unsigned long f : 1; /* Fetch-Protection Bit */
90 unsigned long fc : 1; /* Format-Control */
91 unsigned long p : 1; /* DAT-Protection Bit */
92 unsigned long co : 1; /* Change-Recording Override */
94 unsigned long i : 1; /* Region-Invalid Bit */
95 unsigned long cr : 1; /* Common-Region Bit */
96 unsigned long tt : 2; /* Table-Type Bits */
100 union region3_table_entry {
102 struct region3_table_entry_fc0 fc0;
103 struct region3_table_entry_fc1 fc1;
106 unsigned long fc : 1; /* Format-Control */
108 unsigned long i : 1; /* Region-Invalid Bit */
109 unsigned long cr : 1; /* Common-Region Bit */
110 unsigned long tt : 2; /* Table-Type Bits */
115 struct segment_entry_fc0 {
116 unsigned long pto: 53;/* Page-Table Origin */
117 unsigned long fc : 1; /* Format-Control */
118 unsigned long p : 1; /* DAT-Protection Bit */
120 unsigned long i : 1; /* Segment-Invalid Bit */
121 unsigned long cs : 1; /* Common-Segment Bit */
122 unsigned long tt : 2; /* Table-Type Bits */
126 struct segment_entry_fc1 {
127 unsigned long sfaa : 44; /* Segment-Frame Absolute Address */
129 unsigned long av : 1; /* ACCF-Validity Control */
130 unsigned long acc: 4; /* Access-Control Bits */
131 unsigned long f : 1; /* Fetch-Protection Bit */
132 unsigned long fc : 1; /* Format-Control */
133 unsigned long p : 1; /* DAT-Protection Bit */
134 unsigned long co : 1; /* Change-Recording Override */
136 unsigned long i : 1; /* Segment-Invalid Bit */
137 unsigned long cs : 1; /* Common-Segment Bit */
138 unsigned long tt : 2; /* Table-Type Bits */
142 union segment_table_entry {
144 struct segment_entry_fc0 fc0;
145 struct segment_entry_fc1 fc1;
148 unsigned long fc : 1; /* Format-Control */
150 unsigned long i : 1; /* Segment-Invalid Bit */
151 unsigned long cs : 1; /* Common-Segment Bit */
152 unsigned long tt : 2; /* Table-Type Bits */
158 TABLE_TYPE_SEGMENT = 0,
159 TABLE_TYPE_REGION3 = 1,
160 TABLE_TYPE_REGION2 = 2,
161 TABLE_TYPE_REGION1 = 3
164 union page_table_entry {
167 unsigned long pfra : 52; /* Page-Frame Real Address */
168 unsigned long z : 1; /* Zero Bit */
169 unsigned long i : 1; /* Page-Invalid Bit */
170 unsigned long p : 1; /* DAT-Protection Bit */
176 * vaddress union in order to easily decode a virtual address into its
177 * region first index, region second index etc. parts.
182 unsigned long rfx : 11;
183 unsigned long rsx : 11;
184 unsigned long rtx : 11;
185 unsigned long sx : 11;
186 unsigned long px : 8;
187 unsigned long bx : 12;
190 unsigned long rfx01 : 2;
192 unsigned long rsx01 : 2;
194 unsigned long rtx01 : 2;
196 unsigned long sx01 : 2;
202 * raddress union which will contain the result (real or absolute address)
203 * after a page table walk. The rfaa, sfaa and pfra members are used to
204 * simply assign them the value of a region, segment or page table entry.
208 unsigned long rfaa : 33; /* Region-Frame Absolute Address */
209 unsigned long sfaa : 44; /* Segment-Frame Absolute Address */
210 unsigned long pfra : 52; /* Page-Frame Real Address */
233 unsigned long i : 1; /* ALEN-Invalid Bit */
235 unsigned long fo : 1; /* Fetch-Only Bit */
236 unsigned long p : 1; /* Private Bit */
237 unsigned long alesn : 8; /* Access-List-Entry Sequence Number */
238 unsigned long aleax : 16; /* Access-List-Entry Authorization Index */
241 unsigned long asteo : 25; /* ASN-Second-Table-Entry Origin */
243 unsigned long astesn : 32; /* ASTE Sequence Number */
247 unsigned long i : 1; /* ASX-Invalid Bit */
248 unsigned long ato : 29; /* Authority-Table Origin */
250 unsigned long b : 1; /* Base-Space Bit */
251 unsigned long ax : 16; /* Authorization Index */
252 unsigned long atl : 12; /* Authority-Table Length */
254 unsigned long ca : 1; /* Controlled-ASN Bit */
255 unsigned long ra : 1; /* Reusable-ASN Bit */
256 unsigned long asce : 64; /* Address-Space-Control Element */
257 unsigned long ald : 32;
258 unsigned long astesn : 32;
259 /* .. more fields there */
262 int ipte_lock_held(struct kvm_vcpu *vcpu)
264 if (vcpu->arch.sie_block->eca & ECA_SII) {
267 read_lock(&vcpu->kvm->arch.sca_lock);
268 rc = kvm_s390_get_ipte_control(vcpu->kvm)->kh != 0;
269 read_unlock(&vcpu->kvm->arch.sca_lock);
272 return vcpu->kvm->arch.ipte_lock_count != 0;
275 static void ipte_lock_simple(struct kvm_vcpu *vcpu)
277 union ipte_control old, new, *ic;
279 mutex_lock(&vcpu->kvm->arch.ipte_mutex);
280 vcpu->kvm->arch.ipte_lock_count++;
281 if (vcpu->kvm->arch.ipte_lock_count > 1)
284 read_lock(&vcpu->kvm->arch.sca_lock);
285 ic = kvm_s390_get_ipte_control(vcpu->kvm);
287 old = READ_ONCE(*ic);
289 read_unlock(&vcpu->kvm->arch.sca_lock);
295 } while (cmpxchg(&ic->val, old.val, new.val) != old.val);
296 read_unlock(&vcpu->kvm->arch.sca_lock);
298 mutex_unlock(&vcpu->kvm->arch.ipte_mutex);
301 static void ipte_unlock_simple(struct kvm_vcpu *vcpu)
303 union ipte_control old, new, *ic;
305 mutex_lock(&vcpu->kvm->arch.ipte_mutex);
306 vcpu->kvm->arch.ipte_lock_count--;
307 if (vcpu->kvm->arch.ipte_lock_count)
309 read_lock(&vcpu->kvm->arch.sca_lock);
310 ic = kvm_s390_get_ipte_control(vcpu->kvm);
312 old = READ_ONCE(*ic);
315 } while (cmpxchg(&ic->val, old.val, new.val) != old.val);
316 read_unlock(&vcpu->kvm->arch.sca_lock);
317 wake_up(&vcpu->kvm->arch.ipte_wq);
319 mutex_unlock(&vcpu->kvm->arch.ipte_mutex);
322 static void ipte_lock_siif(struct kvm_vcpu *vcpu)
324 union ipte_control old, new, *ic;
327 read_lock(&vcpu->kvm->arch.sca_lock);
328 ic = kvm_s390_get_ipte_control(vcpu->kvm);
330 old = READ_ONCE(*ic);
332 read_unlock(&vcpu->kvm->arch.sca_lock);
339 } while (cmpxchg(&ic->val, old.val, new.val) != old.val);
340 read_unlock(&vcpu->kvm->arch.sca_lock);
343 static void ipte_unlock_siif(struct kvm_vcpu *vcpu)
345 union ipte_control old, new, *ic;
347 read_lock(&vcpu->kvm->arch.sca_lock);
348 ic = kvm_s390_get_ipte_control(vcpu->kvm);
350 old = READ_ONCE(*ic);
355 } while (cmpxchg(&ic->val, old.val, new.val) != old.val);
356 read_unlock(&vcpu->kvm->arch.sca_lock);
358 wake_up(&vcpu->kvm->arch.ipte_wq);
361 void ipte_lock(struct kvm_vcpu *vcpu)
363 if (vcpu->arch.sie_block->eca & ECA_SII)
364 ipte_lock_siif(vcpu);
366 ipte_lock_simple(vcpu);
369 void ipte_unlock(struct kvm_vcpu *vcpu)
371 if (vcpu->arch.sie_block->eca & ECA_SII)
372 ipte_unlock_siif(vcpu);
374 ipte_unlock_simple(vcpu);
377 static int ar_translation(struct kvm_vcpu *vcpu, union asce *asce, u8 ar,
383 unsigned long ald_addr, authority_table_addr;
391 save_access_regs(vcpu->run->s.regs.acrs);
392 alet.val = vcpu->run->s.regs.acrs[ar];
394 if (ar == 0 || alet.val == 0) {
395 asce->val = vcpu->arch.sie_block->gcr[1];
397 } else if (alet.val == 1) {
398 asce->val = vcpu->arch.sie_block->gcr[7];
403 return PGM_ALET_SPECIFICATION;
406 ald_addr = vcpu->arch.sie_block->gcr[5];
408 ald_addr = vcpu->arch.sie_block->gcr[2];
409 ald_addr &= 0x7fffffc0;
411 rc = read_guest_real(vcpu, ald_addr + 16, &ald.val, sizeof(union ald));
415 if (alet.alen / 8 > ald.all)
416 return PGM_ALEN_TRANSLATION;
418 if (0x7fffffff - ald.alo * 128 < alet.alen * 16)
419 return PGM_ADDRESSING;
421 rc = read_guest_real(vcpu, ald.alo * 128 + alet.alen * 16, &ale,
427 return PGM_ALEN_TRANSLATION;
428 if (ale.alesn != alet.alesn)
429 return PGM_ALE_SEQUENCE;
431 rc = read_guest_real(vcpu, ale.asteo * 64, &aste, sizeof(struct aste));
436 return PGM_ASTE_VALIDITY;
437 if (aste.astesn != ale.astesn)
438 return PGM_ASTE_SEQUENCE;
441 eax = (vcpu->arch.sie_block->gcr[8] >> 16) & 0xffff;
442 if (ale.aleax != eax) {
443 if (eax / 16 > aste.atl)
444 return PGM_EXTENDED_AUTHORITY;
446 authority_table_addr = aste.ato * 4 + eax / 4;
448 rc = read_guest_real(vcpu, authority_table_addr,
454 if ((authority_table & (0x40 >> ((eax & 3) * 2))) == 0)
455 return PGM_EXTENDED_AUTHORITY;
459 if (ale.fo == 1 && mode == GACC_STORE)
460 return PGM_PROTECTION;
462 asce->val = aste.asce;
466 struct trans_exc_code_bits {
467 unsigned long addr : 52; /* Translation-exception Address */
468 unsigned long fsi : 2; /* Access Exception Fetch/Store Indication */
470 unsigned long b56 : 1;
472 unsigned long b60 : 1;
473 unsigned long b61 : 1;
474 unsigned long as : 2; /* ASCE Identifier */
478 FSI_UNKNOWN = 0, /* Unknown wether fetch or store */
479 FSI_STORE = 1, /* Exception was due to store operation */
480 FSI_FETCH = 2 /* Exception was due to fetch operation */
490 static int trans_exc(struct kvm_vcpu *vcpu, int code, unsigned long gva,
491 u8 ar, enum gacc_mode mode, enum prot_type prot)
493 struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm;
494 struct trans_exc_code_bits *tec;
496 memset(pgm, 0, sizeof(*pgm));
498 tec = (struct trans_exc_code_bits *)&pgm->trans_exc_code;
518 case PGM_PAGE_TRANSLATION:
519 case PGM_REGION_FIRST_TRANS:
520 case PGM_REGION_SECOND_TRANS:
521 case PGM_REGION_THIRD_TRANS:
522 case PGM_SEGMENT_TRANSLATION:
524 * op_access_id only applies to MOVE_PAGE -> set bit 61
525 * exc_access_id has to be set to 0 for some instructions. Both
526 * cases have to be handled by the caller.
528 tec->addr = gva >> PAGE_SHIFT;
529 tec->fsi = mode == GACC_STORE ? FSI_STORE : FSI_FETCH;
530 tec->as = psw_bits(vcpu->arch.sie_block->gpsw).as;
532 case PGM_ALEN_TRANSLATION:
533 case PGM_ALE_SEQUENCE:
534 case PGM_ASTE_VALIDITY:
535 case PGM_ASTE_SEQUENCE:
536 case PGM_EXTENDED_AUTHORITY:
538 * We can always store exc_access_id, as it is
539 * undefined for non-ar cases. It is undefined for
540 * most DAT protection exceptions.
542 pgm->exc_access_id = ar;
548 static int get_vcpu_asce(struct kvm_vcpu *vcpu, union asce *asce,
549 unsigned long ga, u8 ar, enum gacc_mode mode)
552 struct psw_bits psw = psw_bits(vcpu->arch.sie_block->gpsw);
560 if ((mode == GACC_IFETCH) && (psw.as != PSW_BITS_AS_HOME))
561 psw.as = PSW_BITS_AS_PRIMARY;
564 case PSW_BITS_AS_PRIMARY:
565 asce->val = vcpu->arch.sie_block->gcr[1];
567 case PSW_BITS_AS_SECONDARY:
568 asce->val = vcpu->arch.sie_block->gcr[7];
570 case PSW_BITS_AS_HOME:
571 asce->val = vcpu->arch.sie_block->gcr[13];
573 case PSW_BITS_AS_ACCREG:
574 rc = ar_translation(vcpu, asce, ar, mode);
576 return trans_exc(vcpu, rc, ga, ar, mode, PROT_TYPE_ALC);
582 static int deref_table(struct kvm *kvm, unsigned long gpa, unsigned long *val)
584 return kvm_read_guest(kvm, gpa, val, sizeof(*val));
588 * guest_translate - translate a guest virtual into a guest absolute address
590 * @gva: guest virtual address
591 * @gpa: points to where guest physical (absolute) address should be stored
592 * @asce: effective asce
593 * @mode: indicates the access mode to be used
595 * Translate a guest virtual address into a guest absolute address by means
596 * of dynamic address translation as specified by the architecture.
597 * If the resulting absolute address is not available in the configuration
598 * an addressing exception is indicated and @gpa will not be changed.
600 * Returns: - zero on success; @gpa contains the resulting absolute address
601 * - a negative value if guest access failed due to e.g. broken
603 * - a positve value if an access exception happened. In this case
604 * the returned value is the program interruption code as defined
605 * by the architecture
607 static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long gva,
608 unsigned long *gpa, const union asce asce,
611 union vaddress vaddr = {.addr = gva};
612 union raddress raddr = {.addr = gva};
613 union page_table_entry pte;
614 int dat_protection = 0;
615 union ctlreg0 ctlreg0;
619 ctlreg0.val = vcpu->arch.sie_block->gcr[0];
620 edat1 = ctlreg0.edat && test_kvm_facility(vcpu->kvm, 8);
621 edat2 = edat1 && test_kvm_facility(vcpu->kvm, 78);
624 ptr = asce.origin * 4096;
626 case ASCE_TYPE_REGION1:
627 if (vaddr.rfx01 > asce.tl)
628 return PGM_REGION_FIRST_TRANS;
629 ptr += vaddr.rfx * 8;
631 case ASCE_TYPE_REGION2:
633 return PGM_ASCE_TYPE;
634 if (vaddr.rsx01 > asce.tl)
635 return PGM_REGION_SECOND_TRANS;
636 ptr += vaddr.rsx * 8;
638 case ASCE_TYPE_REGION3:
639 if (vaddr.rfx || vaddr.rsx)
640 return PGM_ASCE_TYPE;
641 if (vaddr.rtx01 > asce.tl)
642 return PGM_REGION_THIRD_TRANS;
643 ptr += vaddr.rtx * 8;
645 case ASCE_TYPE_SEGMENT:
646 if (vaddr.rfx || vaddr.rsx || vaddr.rtx)
647 return PGM_ASCE_TYPE;
648 if (vaddr.sx01 > asce.tl)
649 return PGM_SEGMENT_TRANSLATION;
654 case ASCE_TYPE_REGION1: {
655 union region1_table_entry rfte;
657 if (kvm_is_error_gpa(vcpu->kvm, ptr))
658 return PGM_ADDRESSING;
659 if (deref_table(vcpu->kvm, ptr, &rfte.val))
662 return PGM_REGION_FIRST_TRANS;
663 if (rfte.tt != TABLE_TYPE_REGION1)
664 return PGM_TRANSLATION_SPEC;
665 if (vaddr.rsx01 < rfte.tf || vaddr.rsx01 > rfte.tl)
666 return PGM_REGION_SECOND_TRANS;
668 dat_protection |= rfte.p;
669 ptr = rfte.rto * 4096 + vaddr.rsx * 8;
672 case ASCE_TYPE_REGION2: {
673 union region2_table_entry rste;
675 if (kvm_is_error_gpa(vcpu->kvm, ptr))
676 return PGM_ADDRESSING;
677 if (deref_table(vcpu->kvm, ptr, &rste.val))
680 return PGM_REGION_SECOND_TRANS;
681 if (rste.tt != TABLE_TYPE_REGION2)
682 return PGM_TRANSLATION_SPEC;
683 if (vaddr.rtx01 < rste.tf || vaddr.rtx01 > rste.tl)
684 return PGM_REGION_THIRD_TRANS;
686 dat_protection |= rste.p;
687 ptr = rste.rto * 4096 + vaddr.rtx * 8;
690 case ASCE_TYPE_REGION3: {
691 union region3_table_entry rtte;
693 if (kvm_is_error_gpa(vcpu->kvm, ptr))
694 return PGM_ADDRESSING;
695 if (deref_table(vcpu->kvm, ptr, &rtte.val))
698 return PGM_REGION_THIRD_TRANS;
699 if (rtte.tt != TABLE_TYPE_REGION3)
700 return PGM_TRANSLATION_SPEC;
701 if (rtte.cr && asce.p && edat2)
702 return PGM_TRANSLATION_SPEC;
703 if (rtte.fc && edat2) {
704 dat_protection |= rtte.fc1.p;
705 raddr.rfaa = rtte.fc1.rfaa;
706 goto absolute_address;
708 if (vaddr.sx01 < rtte.fc0.tf)
709 return PGM_SEGMENT_TRANSLATION;
710 if (vaddr.sx01 > rtte.fc0.tl)
711 return PGM_SEGMENT_TRANSLATION;
713 dat_protection |= rtte.fc0.p;
714 ptr = rtte.fc0.sto * 4096 + vaddr.sx * 8;
717 case ASCE_TYPE_SEGMENT: {
718 union segment_table_entry ste;
720 if (kvm_is_error_gpa(vcpu->kvm, ptr))
721 return PGM_ADDRESSING;
722 if (deref_table(vcpu->kvm, ptr, &ste.val))
725 return PGM_SEGMENT_TRANSLATION;
726 if (ste.tt != TABLE_TYPE_SEGMENT)
727 return PGM_TRANSLATION_SPEC;
728 if (ste.cs && asce.p)
729 return PGM_TRANSLATION_SPEC;
730 if (ste.fc && edat1) {
731 dat_protection |= ste.fc1.p;
732 raddr.sfaa = ste.fc1.sfaa;
733 goto absolute_address;
735 dat_protection |= ste.fc0.p;
736 ptr = ste.fc0.pto * 2048 + vaddr.px * 8;
739 if (kvm_is_error_gpa(vcpu->kvm, ptr))
740 return PGM_ADDRESSING;
741 if (deref_table(vcpu->kvm, ptr, &pte.val))
744 return PGM_PAGE_TRANSLATION;
746 return PGM_TRANSLATION_SPEC;
747 dat_protection |= pte.p;
748 raddr.pfra = pte.pfra;
750 raddr.addr = kvm_s390_real_to_abs(vcpu, raddr.addr);
752 if (mode == GACC_STORE && dat_protection)
753 return PGM_PROTECTION;
754 if (kvm_is_error_gpa(vcpu->kvm, raddr.addr))
755 return PGM_ADDRESSING;
760 static inline int is_low_address(unsigned long ga)
762 /* Check for address ranges 0..511 and 4096..4607 */
763 return (ga & ~0x11fful) == 0;
766 static int low_address_protection_enabled(struct kvm_vcpu *vcpu,
767 const union asce asce)
769 union ctlreg0 ctlreg0 = {.val = vcpu->arch.sie_block->gcr[0]};
770 psw_t *psw = &vcpu->arch.sie_block->gpsw;
774 if (psw_bits(*psw).dat && asce.p)
779 static int guest_page_range(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar,
780 unsigned long *pages, unsigned long nr_pages,
781 const union asce asce, enum gacc_mode mode)
783 psw_t *psw = &vcpu->arch.sie_block->gpsw;
784 int lap_enabled, rc = 0;
786 lap_enabled = low_address_protection_enabled(vcpu, asce);
788 ga = kvm_s390_logical_to_effective(vcpu, ga);
789 if (mode == GACC_STORE && lap_enabled && is_low_address(ga))
790 return trans_exc(vcpu, PGM_PROTECTION, ga, ar, mode,
793 if (psw_bits(*psw).dat) {
794 rc = guest_translate(vcpu, ga, pages, asce, mode);
798 *pages = kvm_s390_real_to_abs(vcpu, ga);
799 if (kvm_is_error_gpa(vcpu->kvm, *pages))
803 return trans_exc(vcpu, rc, ga, ar, mode, PROT_TYPE_DAT);
811 int access_guest(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar, void *data,
812 unsigned long len, enum gacc_mode mode)
814 psw_t *psw = &vcpu->arch.sie_block->gpsw;
815 unsigned long _len, nr_pages, gpa, idx;
816 unsigned long pages_array[2];
817 unsigned long *pages;
824 ga = kvm_s390_logical_to_effective(vcpu, ga);
825 rc = get_vcpu_asce(vcpu, &asce, ga, ar, mode);
828 nr_pages = (((ga & ~PAGE_MASK) + len - 1) >> PAGE_SHIFT) + 1;
830 if (nr_pages > ARRAY_SIZE(pages_array))
831 pages = vmalloc(nr_pages * sizeof(unsigned long));
834 need_ipte_lock = psw_bits(*psw).dat && !asce.r;
837 rc = guest_page_range(vcpu, ga, ar, pages, nr_pages, asce, mode);
838 for (idx = 0; idx < nr_pages && !rc; idx++) {
839 gpa = *(pages + idx) + (ga & ~PAGE_MASK);
840 _len = min(PAGE_SIZE - (gpa & ~PAGE_MASK), len);
841 if (mode == GACC_STORE)
842 rc = kvm_write_guest(vcpu->kvm, gpa, data, _len);
844 rc = kvm_read_guest(vcpu->kvm, gpa, data, _len);
851 if (nr_pages > ARRAY_SIZE(pages_array))
856 int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra,
857 void *data, unsigned long len, enum gacc_mode mode)
859 unsigned long _len, gpa;
863 gpa = kvm_s390_real_to_abs(vcpu, gra);
864 _len = min(PAGE_SIZE - (gpa & ~PAGE_MASK), len);
866 rc = write_guest_abs(vcpu, gpa, data, _len);
868 rc = read_guest_abs(vcpu, gpa, data, _len);
877 * guest_translate_address - translate guest logical into guest absolute address
879 * Parameter semantics are the same as the ones from guest_translate.
880 * The memory contents at the guest address are not changed.
882 * Note: The IPTE lock is not taken during this function, so the caller
883 * has to take care of this.
885 int guest_translate_address(struct kvm_vcpu *vcpu, unsigned long gva, u8 ar,
886 unsigned long *gpa, enum gacc_mode mode)
888 psw_t *psw = &vcpu->arch.sie_block->gpsw;
892 gva = kvm_s390_logical_to_effective(vcpu, gva);
893 rc = get_vcpu_asce(vcpu, &asce, gva, ar, mode);
896 if (is_low_address(gva) && low_address_protection_enabled(vcpu, asce)) {
897 if (mode == GACC_STORE)
898 return trans_exc(vcpu, PGM_PROTECTION, gva, 0,
902 if (psw_bits(*psw).dat && !asce.r) { /* Use DAT? */
903 rc = guest_translate(vcpu, gva, gpa, asce, mode);
905 return trans_exc(vcpu, rc, gva, 0, mode, PROT_TYPE_DAT);
907 *gpa = kvm_s390_real_to_abs(vcpu, gva);
908 if (kvm_is_error_gpa(vcpu->kvm, *gpa))
909 return trans_exc(vcpu, rc, gva, PGM_ADDRESSING, mode, 0);
916 * check_gva_range - test a range of guest virtual addresses for accessibility
918 int check_gva_range(struct kvm_vcpu *vcpu, unsigned long gva, u8 ar,
919 unsigned long length, enum gacc_mode mode)
922 unsigned long currlen;
926 while (length > 0 && !rc) {
927 currlen = min(length, PAGE_SIZE - (gva % PAGE_SIZE));
928 rc = guest_translate_address(vcpu, gva, ar, &gpa, mode);
938 * kvm_s390_check_low_addr_prot_real - check for low-address protection
939 * @gra: Guest real address
941 * Checks whether an address is subject to low-address protection and set
942 * up vcpu->arch.pgm accordingly if necessary.
944 * Return: 0 if no protection exception, or PGM_PROTECTION if protected.
946 int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra)
948 union ctlreg0 ctlreg0 = {.val = vcpu->arch.sie_block->gcr[0]};
950 if (!ctlreg0.lap || !is_low_address(gra))
952 return trans_exc(vcpu, PGM_PROTECTION, gra, 0, GACC_STORE, PROT_TYPE_LA);
956 * kvm_s390_shadow_tables - walk the guest page table and create shadow tables
957 * @sg: pointer to the shadow guest address space structure
958 * @saddr: faulting address in the shadow gmap
959 * @pgt: pointer to the page table address result
960 * @fake: pgt references contiguous guest memory block, not a pgtable
962 static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
963 unsigned long *pgt, int *dat_protection,
968 union vaddress vaddr;
976 asce.val = sg->orig_asce;
977 ptr = asce.origin * 4096;
980 asce.dt = ASCE_TYPE_REGION1;
983 case ASCE_TYPE_REGION1:
984 if (vaddr.rfx01 > asce.tl && !asce.r)
985 return PGM_REGION_FIRST_TRANS;
987 case ASCE_TYPE_REGION2:
989 return PGM_ASCE_TYPE;
990 if (vaddr.rsx01 > asce.tl)
991 return PGM_REGION_SECOND_TRANS;
993 case ASCE_TYPE_REGION3:
994 if (vaddr.rfx || vaddr.rsx)
995 return PGM_ASCE_TYPE;
996 if (vaddr.rtx01 > asce.tl)
997 return PGM_REGION_THIRD_TRANS;
999 case ASCE_TYPE_SEGMENT:
1000 if (vaddr.rfx || vaddr.rsx || vaddr.rtx)
1001 return PGM_ASCE_TYPE;
1002 if (vaddr.sx01 > asce.tl)
1003 return PGM_SEGMENT_TRANSLATION;
1008 case ASCE_TYPE_REGION1: {
1009 union region1_table_entry rfte;
1012 /* offset in 16EB guest memory block */
1013 ptr = ptr + ((unsigned long) vaddr.rsx << 53UL);
1017 rc = gmap_read_table(parent, ptr + vaddr.rfx * 8, &rfte.val);
1021 return PGM_REGION_FIRST_TRANS;
1022 if (rfte.tt != TABLE_TYPE_REGION1)
1023 return PGM_TRANSLATION_SPEC;
1024 if (vaddr.rsx01 < rfte.tf || vaddr.rsx01 > rfte.tl)
1025 return PGM_REGION_SECOND_TRANS;
1026 if (sg->edat_level >= 1)
1027 *dat_protection |= rfte.p;
1028 ptr = rfte.rto << 12UL;
1030 rc = gmap_shadow_r2t(sg, saddr, rfte.val, *fake);
1035 case ASCE_TYPE_REGION2: {
1036 union region2_table_entry rste;
1039 /* offset in 8PB guest memory block */
1040 ptr = ptr + ((unsigned long) vaddr.rtx << 42UL);
1044 rc = gmap_read_table(parent, ptr + vaddr.rsx * 8, &rste.val);
1048 return PGM_REGION_SECOND_TRANS;
1049 if (rste.tt != TABLE_TYPE_REGION2)
1050 return PGM_TRANSLATION_SPEC;
1051 if (vaddr.rtx01 < rste.tf || vaddr.rtx01 > rste.tl)
1052 return PGM_REGION_THIRD_TRANS;
1053 if (sg->edat_level >= 1)
1054 *dat_protection |= rste.p;
1055 ptr = rste.rto << 12UL;
1057 rste.p |= *dat_protection;
1058 rc = gmap_shadow_r3t(sg, saddr, rste.val, *fake);
1063 case ASCE_TYPE_REGION3: {
1064 union region3_table_entry rtte;
1067 /* offset in 4TB guest memory block */
1068 ptr = ptr + ((unsigned long) vaddr.sx << 31UL);
1072 rc = gmap_read_table(parent, ptr + vaddr.rtx * 8, &rtte.val);
1076 return PGM_REGION_THIRD_TRANS;
1077 if (rtte.tt != TABLE_TYPE_REGION3)
1078 return PGM_TRANSLATION_SPEC;
1079 if (rtte.cr && asce.p && sg->edat_level >= 2)
1080 return PGM_TRANSLATION_SPEC;
1081 if (rtte.fc && sg->edat_level >= 2) {
1082 *dat_protection |= rtte.fc0.p;
1084 ptr = rtte.fc1.rfaa << 31UL;
1088 if (vaddr.sx01 < rtte.fc0.tf || vaddr.sx01 > rtte.fc0.tl)
1089 return PGM_SEGMENT_TRANSLATION;
1090 if (sg->edat_level >= 1)
1091 *dat_protection |= rtte.fc0.p;
1092 ptr = rtte.fc0.sto << 12UL;
1094 rtte.fc0.p |= *dat_protection;
1095 rc = gmap_shadow_sgt(sg, saddr, rtte.val, *fake);
1100 case ASCE_TYPE_SEGMENT: {
1101 union segment_table_entry ste;
1104 /* offset in 2G guest memory block */
1105 ptr = ptr + ((unsigned long) vaddr.sx << 20UL);
1109 rc = gmap_read_table(parent, ptr + vaddr.sx * 8, &ste.val);
1113 return PGM_SEGMENT_TRANSLATION;
1114 if (ste.tt != TABLE_TYPE_SEGMENT)
1115 return PGM_TRANSLATION_SPEC;
1116 if (ste.cs && asce.p)
1117 return PGM_TRANSLATION_SPEC;
1118 *dat_protection |= ste.fc0.p;
1119 if (ste.fc && sg->edat_level >= 1) {
1121 ptr = ste.fc1.sfaa << 20UL;
1125 ptr = ste.fc0.pto << 11UL;
1127 ste.fc0.p |= *dat_protection;
1128 rc = gmap_shadow_pgt(sg, saddr, ste.val, *fake);
1133 /* Return the parent address of the page table */
1139 * kvm_s390_shadow_fault - handle fault on a shadow page table
1140 * @vcpu: virtual cpu
1141 * @sg: pointer to the shadow guest address space structure
1142 * @saddr: faulting address in the shadow gmap
1144 * Returns: - 0 if the shadow fault was successfully resolved
1145 * - > 0 (pgm exception code) on exceptions while faulting
1146 * - -EAGAIN if the caller can retry immediately
1147 * - -EFAULT when accessing invalid guest addresses
1148 * - -ENOMEM if out of memory
1150 int kvm_s390_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg,
1151 unsigned long saddr)
1153 union vaddress vaddr;
1154 union page_table_entry pte;
1156 int dat_protection, fake;
1159 down_read(&sg->mm->mmap_sem);
1161 * We don't want any guest-2 tables to change - so the parent
1162 * tables/pointers we read stay valid - unshadowing is however
1163 * always possible - only guest_table_lock protects us.
1167 rc = gmap_shadow_pgt_lookup(sg, saddr, &pgt, &dat_protection, &fake);
1169 rc = kvm_s390_shadow_tables(sg, saddr, &pgt, &dat_protection,
1174 /* offset in 1MB guest memory block */
1175 pte.val = pgt + ((unsigned long) vaddr.px << 12UL);
1179 rc = gmap_read_table(sg->parent, pgt + vaddr.px * 8, &pte.val);
1181 rc = PGM_PAGE_TRANSLATION;
1183 rc = PGM_TRANSLATION_SPEC;
1185 pte.p |= dat_protection;
1187 rc = gmap_shadow_page(sg, saddr, __pte(pte.val));
1189 up_read(&sg->mm->mmap_sem);