config XIP_KERNEL
bool "Kernel Execute-In-Place from ROM"
- depends on !ZBOOT_ROM
+ depends on !ZBOOT_ROM && !ARM_LPAE
help
Execute-In-Place allows the kernel to run from non-volatile storage
directly addressable by the CPU, such as NOR flash. This saves RAM
mcrne p15, 0, r3, c2, c0, 0 @ load page table pointer
mcrne p15, 0, r1, c3, c0, 0 @ load domain access control
#endif
+ mcr p15, 0, r0, c7, c5, 4 @ ISB
mcr p15, 0, r0, c1, c0, 0 @ load control register
mrc p15, 0, r0, c1, c0, 0 @ and read it back
mov r0, #0
#define ALT_UP_B(label) b label
#endif
+/*
+ * Instruction barrier
+ */
+ .macro instr_sync
+#if __LINUX_ARM_ARCH__ >= 7
+ isb
+#elif __LINUX_ARM_ARCH__ == 6
+ mcr p15, 0, r0, c7, c5, 4
+#endif
+ .endm
+
/*
* SMP data memory barrier
*/
#define clear_page(page) memset((void *)(page), 0, PAGE_SIZE)
extern void copy_page(void *to, const void *from);
+#ifdef CONFIG_ARM_LPAE
+#include <asm/pgtable-3level-types.h>
+#else
#include <asm/pgtable-2level-types.h>
+#endif
#endif /* CONFIG_MMU */
#define _PAGE_USER_TABLE (PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_USER))
#define _PAGE_KERNEL_TABLE (PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_KERNEL))
+#ifdef CONFIG_ARM_LPAE
+
+static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
+{
+ return (pmd_t *)get_zeroed_page(GFP_KERNEL | __GFP_REPEAT);
+}
+
+static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
+{
+ BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
+ free_page((unsigned long)pmd);
+}
+
+static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
+{
+ set_pud(pud, __pud(__pa(pmd) | PMD_TYPE_TABLE));
+}
+
+#else /* !CONFIG_ARM_LPAE */
+
/*
* Since we have only two-level page tables, these are trivial
*/
#define pmd_alloc_one(mm,addr) ({ BUG(); ((pmd_t *)2); })
#define pmd_free(mm, pmd) do { } while (0)
-#define pgd_populate(mm,pmd,pte) BUG()
+#define pud_populate(mm,pmd,pte) BUG()
+
+#endif /* CONFIG_ARM_LPAE */
extern pgd_t *pgd_alloc(struct mm_struct *mm);
extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
{
pmdval_t pmdval = (pte + PTE_HWTABLE_OFF) | prot;
pmdp[0] = __pmd(pmdval);
+#ifndef CONFIG_ARM_LPAE
pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t));
+#endif
flush_pmd_entry(pmdp);
}
#define L_PTE_MT_DEV_CACHED (_AT(pteval_t, 0x0b) << 2) /* 1011 */
#define L_PTE_MT_MASK (_AT(pteval_t, 0x0f) << 2)
+#ifndef __ASSEMBLY__
+
+/*
+ * The "pud_xxx()" functions here are trivial when the pmd is folded into
+ * the pud: the pud entry is never bad, always exists, and can't be set or
+ * cleared.
+ */
+#define pud_none(pud) (0)
+#define pud_bad(pud) (0)
+#define pud_present(pud) (1)
+#define pud_clear(pudp) do { } while (0)
+#define set_pud(pud,pudp) do { } while (0)
+
+static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr)
+{
+ return (pmd_t *)pud;
+}
+
+#define pmd_bad(pmd) (pmd_val(pmd) & 2)
+
+#define copy_pmd(pmdpd,pmdps) \
+ do { \
+ pmdpd[0] = pmdps[0]; \
+ pmdpd[1] = pmdps[1]; \
+ flush_pmd_entry(pmdpd); \
+ } while (0)
+
+#define pmd_clear(pmdp) \
+ do { \
+ pmdp[0] = __pmd(0); \
+ pmdp[1] = __pmd(0); \
+ clean_pmd_entry(pmdp); \
+ } while (0)
+
+/* we don't need complex calculations here as the pmd is folded into the pgd */
+#define pmd_addr_end(addr,end) (end)
+
+#define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,pte,ext)
+
+#endif /* __ASSEMBLY__ */
+
#endif /* _ASM_PGTABLE_2LEVEL_H */
--- /dev/null
+/*
+ * arch/arm/include/asm/pgtable-3level-hwdef.h
+ *
+ * Copyright (C) 2011 ARM Ltd.
+ * Author: Catalin Marinas <catalin.marinas@arm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+#ifndef _ASM_PGTABLE_3LEVEL_HWDEF_H
+#define _ASM_PGTABLE_3LEVEL_HWDEF_H
+
+/*
+ * Hardware page table definitions.
+ *
+ * + Level 1/2 descriptor
+ * - common
+ */
+#define PMD_TYPE_MASK (_AT(pmdval_t, 3) << 0)
+#define PMD_TYPE_FAULT (_AT(pmdval_t, 0) << 0)
+#define PMD_TYPE_TABLE (_AT(pmdval_t, 3) << 0)
+#define PMD_TYPE_SECT (_AT(pmdval_t, 1) << 0)
+#define PMD_BIT4 (_AT(pmdval_t, 0))
+#define PMD_DOMAIN(x) (_AT(pmdval_t, 0))
+
+/*
+ * - section
+ */
+#define PMD_SECT_BUFFERABLE (_AT(pmdval_t, 1) << 2)
+#define PMD_SECT_CACHEABLE (_AT(pmdval_t, 1) << 3)
+#define PMD_SECT_S (_AT(pmdval_t, 3) << 8)
+#define PMD_SECT_AF (_AT(pmdval_t, 1) << 10)
+#define PMD_SECT_nG (_AT(pmdval_t, 1) << 11)
+#define PMD_SECT_XN (_AT(pmdval_t, 1) << 54)
+#define PMD_SECT_AP_WRITE (_AT(pmdval_t, 0))
+#define PMD_SECT_AP_READ (_AT(pmdval_t, 0))
+#define PMD_SECT_TEX(x) (_AT(pmdval_t, 0))
+
+/*
+ * AttrIndx[2:0] encoding (mapping attributes defined in the MAIR* registers).
+ */
+#define PMD_SECT_UNCACHED (_AT(pmdval_t, 0) << 2) /* strongly ordered */
+#define PMD_SECT_BUFFERED (_AT(pmdval_t, 1) << 2) /* normal non-cacheable */
+#define PMD_SECT_WT (_AT(pmdval_t, 2) << 2) /* normal inner write-through */
+#define PMD_SECT_WB (_AT(pmdval_t, 3) << 2) /* normal inner write-back */
+#define PMD_SECT_WBWA (_AT(pmdval_t, 7) << 2) /* normal inner write-alloc */
+
+/*
+ * + Level 3 descriptor (PTE)
+ */
+#define PTE_TYPE_MASK (_AT(pteval_t, 3) << 0)
+#define PTE_TYPE_FAULT (_AT(pteval_t, 0) << 0)
+#define PTE_TYPE_PAGE (_AT(pteval_t, 3) << 0)
+#define PTE_BUFFERABLE (_AT(pteval_t, 1) << 2) /* AttrIndx[0] */
+#define PTE_CACHEABLE (_AT(pteval_t, 1) << 3) /* AttrIndx[1] */
+#define PTE_EXT_SHARED (_AT(pteval_t, 3) << 8) /* SH[1:0], inner shareable */
+#define PTE_EXT_AF (_AT(pteval_t, 1) << 10) /* Access Flag */
+#define PTE_EXT_NG (_AT(pteval_t, 1) << 11) /* nG */
+#define PTE_EXT_XN (_AT(pteval_t, 1) << 54) /* XN */
+
+/*
+ * 40-bit physical address supported.
+ */
+#define PHYS_MASK_SHIFT (40)
+#define PHYS_MASK ((1ULL << PHYS_MASK_SHIFT) - 1)
+
+#endif
--- /dev/null
+/*
+ * arch/arm/include/asm/pgtable-3level-types.h
+ *
+ * Copyright (C) 2011 ARM Ltd.
+ * Author: Catalin Marinas <catalin.marinas@arm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+#ifndef _ASM_PGTABLE_3LEVEL_TYPES_H
+#define _ASM_PGTABLE_3LEVEL_TYPES_H
+
+#include <asm/types.h>
+
+typedef u64 pteval_t;
+typedef u64 pmdval_t;
+typedef u64 pgdval_t;
+
+#undef STRICT_MM_TYPECHECKS
+
+#ifdef STRICT_MM_TYPECHECKS
+
+/*
+ * These are used to make use of C type-checking..
+ */
+typedef struct { pteval_t pte; } pte_t;
+typedef struct { pmdval_t pmd; } pmd_t;
+typedef struct { pgdval_t pgd; } pgd_t;
+typedef struct { pteval_t pgprot; } pgprot_t;
+
+#define pte_val(x) ((x).pte)
+#define pmd_val(x) ((x).pmd)
+#define pgd_val(x) ((x).pgd)
+#define pgprot_val(x) ((x).pgprot)
+
+#define __pte(x) ((pte_t) { (x) } )
+#define __pmd(x) ((pmd_t) { (x) } )
+#define __pgd(x) ((pgd_t) { (x) } )
+#define __pgprot(x) ((pgprot_t) { (x) } )
+
+#else /* !STRICT_MM_TYPECHECKS */
+
+typedef pteval_t pte_t;
+typedef pmdval_t pmd_t;
+typedef pgdval_t pgd_t;
+typedef pteval_t pgprot_t;
+
+#define pte_val(x) (x)
+#define pmd_val(x) (x)
+#define pgd_val(x) (x)
+#define pgprot_val(x) (x)
+
+#define __pte(x) (x)
+#define __pmd(x) (x)
+#define __pgd(x) (x)
+#define __pgprot(x) (x)
+
+#endif /* STRICT_MM_TYPECHECKS */
+
+#endif /* _ASM_PGTABLE_3LEVEL_TYPES_H */
--- /dev/null
+/*
+ * arch/arm/include/asm/pgtable-3level.h
+ *
+ * Copyright (C) 2011 ARM Ltd.
+ * Author: Catalin Marinas <catalin.marinas@arm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+#ifndef _ASM_PGTABLE_3LEVEL_H
+#define _ASM_PGTABLE_3LEVEL_H
+
+/*
+ * With LPAE, there are 3 levels of page tables. Each level has 512 entries of
+ * 8 bytes each, occupying a 4K page. The first level table covers a range of
+ * 512GB, each entry representing 1GB. Since we are limited to 4GB input
+ * address range, only 4 entries in the PGD are used.
+ *
+ * There are enough spare bits in a page table entry for the kernel specific
+ * state.
+ */
+#define PTRS_PER_PTE 512
+#define PTRS_PER_PMD 512
+#define PTRS_PER_PGD 4
+
+#define PTE_HWTABLE_PTRS (PTRS_PER_PTE)
+#define PTE_HWTABLE_OFF (0)
+#define PTE_HWTABLE_SIZE (PTRS_PER_PTE * sizeof(u64))
+
+/*
+ * PGDIR_SHIFT determines the size a top-level page table entry can map.
+ */
+#define PGDIR_SHIFT 30
+
+/*
+ * PMD_SHIFT determines the size a middle-level page table entry can map.
+ */
+#define PMD_SHIFT 21
+
+#define PMD_SIZE (1UL << PMD_SHIFT)
+#define PMD_MASK (~(PMD_SIZE-1))
+#define PGDIR_SIZE (1UL << PGDIR_SHIFT)
+#define PGDIR_MASK (~(PGDIR_SIZE-1))
+
+/*
+ * section address mask and size definitions.
+ */
+#define SECTION_SHIFT 21
+#define SECTION_SIZE (1UL << SECTION_SHIFT)
+#define SECTION_MASK (~(SECTION_SIZE-1))
+
+#define USER_PTRS_PER_PGD (PAGE_OFFSET / PGDIR_SIZE)
+
+/*
+ * "Linux" PTE definitions for LPAE.
+ *
+ * These bits overlap with the hardware bits but the naming is preserved for
+ * consistency with the classic page table format.
+ */
+#define L_PTE_PRESENT (_AT(pteval_t, 3) << 0) /* Valid */
+#define L_PTE_FILE (_AT(pteval_t, 1) << 2) /* only when !PRESENT */
+#define L_PTE_BUFFERABLE (_AT(pteval_t, 1) << 2) /* AttrIndx[0] */
+#define L_PTE_CACHEABLE (_AT(pteval_t, 1) << 3) /* AttrIndx[1] */
+#define L_PTE_USER (_AT(pteval_t, 1) << 6) /* AP[1] */
+#define L_PTE_RDONLY (_AT(pteval_t, 1) << 7) /* AP[2] */
+#define L_PTE_SHARED (_AT(pteval_t, 3) << 8) /* SH[1:0], inner shareable */
+#define L_PTE_YOUNG (_AT(pteval_t, 1) << 10) /* AF */
+#define L_PTE_XN (_AT(pteval_t, 1) << 54) /* XN */
+#define L_PTE_DIRTY (_AT(pteval_t, 1) << 55) /* unused */
+#define L_PTE_SPECIAL (_AT(pteval_t, 1) << 56) /* unused */
+
+/*
+ * To be used in assembly code with the upper page attributes.
+ */
+#define L_PTE_XN_HIGH (1 << (54 - 32))
+#define L_PTE_DIRTY_HIGH (1 << (55 - 32))
+
+/*
+ * AttrIndx[2:0] encoding (mapping attributes defined in the MAIR* registers).
+ */
+#define L_PTE_MT_UNCACHED (_AT(pteval_t, 0) << 2) /* strongly ordered */
+#define L_PTE_MT_BUFFERABLE (_AT(pteval_t, 1) << 2) /* normal non-cacheable */
+#define L_PTE_MT_WRITETHROUGH (_AT(pteval_t, 2) << 2) /* normal inner write-through */
+#define L_PTE_MT_WRITEBACK (_AT(pteval_t, 3) << 2) /* normal inner write-back */
+#define L_PTE_MT_WRITEALLOC (_AT(pteval_t, 7) << 2) /* normal inner write-alloc */
+#define L_PTE_MT_DEV_SHARED (_AT(pteval_t, 4) << 2) /* device */
+#define L_PTE_MT_DEV_NONSHARED (_AT(pteval_t, 4) << 2) /* device */
+#define L_PTE_MT_DEV_WC (_AT(pteval_t, 1) << 2) /* normal non-cacheable */
+#define L_PTE_MT_DEV_CACHED (_AT(pteval_t, 3) << 2) /* normal inner write-back */
+#define L_PTE_MT_MASK (_AT(pteval_t, 7) << 2)
+
+/*
+ * Software PGD flags.
+ */
+#define L_PGD_SWAPPER (_AT(pgdval_t, 1) << 55) /* swapper_pg_dir entry */
+
+#ifndef __ASSEMBLY__
+
+#define pud_none(pud) (!pud_val(pud))
+#define pud_bad(pud) (!(pud_val(pud) & 2))
+#define pud_present(pud) (pud_val(pud))
+
+#define pud_clear(pudp) \
+ do { \
+ *pudp = __pud(0); \
+ clean_pmd_entry(pudp); \
+ } while (0)
+
+#define set_pud(pudp, pud) \
+ do { \
+ *pudp = pud; \
+ flush_pmd_entry(pudp); \
+ } while (0)
+
+static inline pmd_t *pud_page_vaddr(pud_t pud)
+{
+ return __va(pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK);
+}
+
+/* Find an entry in the second-level page table.. */
+#define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
+static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr)
+{
+ return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(addr);
+}
+
+#define pmd_bad(pmd) (!(pmd_val(pmd) & 2))
+
+#define copy_pmd(pmdpd,pmdps) \
+ do { \
+ *pmdpd = *pmdps; \
+ flush_pmd_entry(pmdpd); \
+ } while (0)
+
+#define pmd_clear(pmdp) \
+ do { \
+ *pmdp = __pmd(0); \
+ clean_pmd_entry(pmdp); \
+ } while (0)
+
+#define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,__pte(pte_val(pte)|(ext)))
+
+#endif /* __ASSEMBLY__ */
+
+#endif /* _ASM_PGTABLE_3LEVEL_H */
#ifndef _ASMARM_PGTABLE_HWDEF_H
#define _ASMARM_PGTABLE_HWDEF_H
+#ifdef CONFIG_ARM_LPAE
+#include <asm/pgtable-3level-hwdef.h>
+#else
#include <asm/pgtable-2level-hwdef.h>
+#endif
#endif
#define _ASMARM_PGTABLE_H
#include <linux/const.h>
-#include <asm-generic/4level-fixup.h>
#include <asm/proc-fns.h>
#ifndef CONFIG_MMU
+#include <asm-generic/4level-fixup.h>
#include "pgtable-nommu.h"
#else
+#include <asm-generic/pgtable-nopud.h>
#include <asm/memory.h>
#include <asm/pgtable-hwdef.h>
+#ifdef CONFIG_ARM_LPAE
+#include <asm/pgtable-3level.h>
+#else
#include <asm/pgtable-2level.h>
+#endif
/*
* Just any arbitrary offset to the start of the vmalloc VM area: the
/* to find an entry in a kernel page-table-directory */
#define pgd_offset_k(addr) pgd_offset(&init_mm, addr)
-/*
- * The "pgd_xxx()" functions here are trivial for a folded two-level
- * setup: the pgd is never bad, and a pmd always exists (as it's folded
- * into the pgd entry)
- */
-#define pgd_none(pgd) (0)
-#define pgd_bad(pgd) (0)
-#define pgd_present(pgd) (1)
-#define pgd_clear(pgdp) do { } while (0)
-#define set_pgd(pgd,pgdp) do { } while (0)
-#define set_pud(pud,pudp) do { } while (0)
-
-
-/* Find an entry in the second-level page table.. */
-#define pmd_offset(dir, addr) ((pmd_t *)(dir))
-
#define pmd_none(pmd) (!pmd_val(pmd))
#define pmd_present(pmd) (pmd_val(pmd))
-#define pmd_bad(pmd) (pmd_val(pmd) & 2)
-
-#define copy_pmd(pmdpd,pmdps) \
- do { \
- pmdpd[0] = pmdps[0]; \
- pmdpd[1] = pmdps[1]; \
- flush_pmd_entry(pmdpd); \
- } while (0)
-
-#define pmd_clear(pmdp) \
- do { \
- pmdp[0] = __pmd(0); \
- pmdp[1] = __pmd(0); \
- clean_pmd_entry(pmdp); \
- } while (0)
static inline pte_t *pmd_page_vaddr(pmd_t pmd)
{
#define pmd_page(pmd) pfn_to_page(__phys_to_pfn(pmd_val(pmd) & PHYS_MASK))
-/* we don't need complex calculations here as the pmd is folded into the pgd */
-#define pmd_addr_end(addr,end) (end)
-
-
#ifndef CONFIG_HIGHPTE
#define __pte_map(pmd) pmd_page_vaddr(*(pmd))
#define __pte_unmap(pte) do { } while (0)
#define pte_page(pte) pfn_to_page(pte_pfn(pte))
#define mk_pte(page,prot) pfn_pte(page_to_pfn(page), prot)
-#define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,pte,ext)
#define pte_clear(mm,addr,ptep) set_pte_ext(ptep, __pte(0), 0)
#if __LINUX_ARM_ARCH__ < 6
* Set a possibly extended PTE. Non-extended PTEs should
* ignore 'ext'.
*/
+#ifdef CONFIG_ARM_LPAE
+ void (*set_pte_ext)(pte_t *ptep, pte_t pte);
+#else
void (*set_pte_ext)(pte_t *ptep, pte_t pte, unsigned int ext);
+#endif
/* Suspend/resume */
unsigned int suspend_size;
extern int cpu_do_idle(void);
extern void cpu_dcache_clean_area(void *, int);
extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm);
+#ifdef CONFIG_ARM_LPAE
+extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte);
+#else
extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext);
+#endif
extern void cpu_reset(unsigned long addr) __attribute__((noreturn));
/* These three are private to arch/arm/kernel/suspend.c */
#define cpu_switch_mm(pgd,mm) cpu_do_switch_mm(virt_to_phys(pgd),mm)
+#ifdef CONFIG_ARM_LPAE
+#define cpu_get_pgd() \
+ ({ \
+ unsigned long pg, pg2; \
+ __asm__("mrrc p15, 0, %0, %1, c2" \
+ : "=r" (pg), "=r" (pg2) \
+ : \
+ : "cc"); \
+ pg &= ~(PTRS_PER_PGD*sizeof(pgd_t)-1); \
+ (pgd_t *)phys_to_virt(pg); \
+ })
+#else
#define cpu_get_pgd() \
({ \
unsigned long pg; \
pg &= ~0x3fff; \
(pgd_t *)phys_to_virt(pg); \
})
+#endif
#endif
void arm_notify_die(const char *str, struct pt_regs *regs, struct siginfo *info,
unsigned long err, unsigned long trap);
+#ifdef CONFIG_ARM_LPAE
+#define FAULT_CODE_ALIGNMENT 33
+#define FAULT_CODE_DEBUG 34
+#else
+#define FAULT_CODE_ALIGNMENT 1
+#define FAULT_CODE_DEBUG 2
+#endif
+
void hook_fault_code(int nr, int (*fn)(unsigned long, unsigned int,
struct pt_regs *),
int sig, int code, const char *name);
tlb_remove_page(tlb, pte);
}
+static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
+ unsigned long addr)
+{
+#ifdef CONFIG_ARM_LPAE
+ tlb_add_flush(tlb, addr);
+ tlb_remove_page(tlb, virt_to_page(pmdp));
+#endif
+}
+
#define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr)
-#define pmd_free_tlb(tlb, pmdp, addr) pmd_free((tlb)->mm, pmdp)
+#define pmd_free_tlb(tlb, pmdp, addr) __pmd_free_tlb(tlb, pmdp, addr)
+#define pud_free_tlb(tlb, pudp, addr) pud_free((tlb)->mm, pudp)
#define tlb_migrate_finish(mm) do { } while (0)
#error KERNEL_RAM_VADDR must start at 0xXXXX8000
#endif
+#ifdef CONFIG_ARM_LPAE
+ /* LPAE requires an additional page for the PGD */
+#define PG_DIR_SIZE 0x5000
+#define PMD_ORDER 3
+#else
#define PG_DIR_SIZE 0x4000
#define PMD_ORDER 2
+#endif
.globl swapper_pg_dir
.equ swapper_pg_dir, KERNEL_RAM_VADDR - PG_DIR_SIZE
teq r0, r6
bne 1b
+#ifdef CONFIG_ARM_LPAE
+ /*
+ * Build the PGD table (first level) to point to the PMD table. A PGD
+ * entry is 64-bit wide.
+ */
+ mov r0, r4
+ add r3, r4, #0x1000 @ first PMD table address
+ orr r3, r3, #3 @ PGD block type
+ mov r6, #4 @ PTRS_PER_PGD
+ mov r7, #1 << (55 - 32) @ L_PGD_SWAPPER
+1: str r3, [r0], #4 @ set bottom PGD entry bits
+ str r7, [r0], #4 @ set top PGD entry bits
+ add r3, r3, #0x1000 @ next PMD table
+ subs r6, r6, #1
+ bne 1b
+
+ add r4, r4, #0x1000 @ point to the PMD tables
+#endif
+
ldr r7, [r10, #PROCINFO_MM_MMUFLAGS] @ mm_mmuflags
/*
#endif
/*
- * Then map boot params address in r2 or
- * the first 1MB of ram if boot params address is not specified.
+ * Then map boot params address in r2 or the first 1MB (2MB with LPAE)
+ * of ram if boot params address is not specified.
*/
mov r0, r2, lsr #SECTION_SHIFT
movs r0, r0, lsl #SECTION_SHIFT
mov r3, r7, lsr #SECTION_SHIFT
ldr r7, [r10, #PROCINFO_IO_MMUFLAGS] @ io_mmuflags
orr r3, r7, r3, lsl #SECTION_SHIFT
+#ifdef CONFIG_ARM_LPAE
+ mov r7, #1 << (54 - 32) @ XN
+#else
+ orr r3, r3, #PMD_SECT_XN
+#endif
1: str r3, [r0], #4
+#ifdef CONFIG_ARM_LPAE
+ str r7, [r0], #4
+#endif
add r3, r3, #1 << SECTION_SHIFT
cmp r0, r6
blo 1b
add r0, r4, #0xd8000000 >> (SECTION_SHIFT - PMD_ORDER)
str r3, [r0]
#endif
+#endif
+#ifdef CONFIG_ARM_LPAE
+ sub r4, r4, #0x1000 @ point to the PGD table
#endif
mov pc, lr
ENDPROC(__create_page_tables)
#ifdef CONFIG_CPU_ICACHE_DISABLE
bic r0, r0, #CR_I
#endif
+#ifdef CONFIG_ARM_LPAE
+ mov r5, #0
+ mcrr p15, 0, r4, r5, c2 @ load TTBR0
+#else
mov r5, #(domain_val(DOMAIN_USER, DOMAIN_MANAGER) | \
domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \
domain_val(DOMAIN_TABLE, DOMAIN_MANAGER) | \
domain_val(DOMAIN_IO, DOMAIN_CLIENT))
mcr p15, 0, r5, c3, c0, 0 @ load domain access register
mcr p15, 0, r4, c2, c0, 0 @ load page table pointer
+#endif
b __turn_mmu_on
ENDPROC(__enable_mmu)
.pushsection .idmap.text, "ax"
ENTRY(__turn_mmu_on)
mov r0, r0
+ instr_sync
mcr p15, 0, r0, c1, c0, 0 @ write control reg
mrc p15, 0, r3, c0, c0, 0 @ read id reg
+ instr_sync
mov r3, r3
mov r3, r13
mov pc, r3
}
/* Register debug fault handler. */
- hook_fault_code(2, hw_breakpoint_pending, SIGTRAP, TRAP_HWBKPT,
- "watchpoint debug exception");
- hook_ifault_code(2, hw_breakpoint_pending, SIGTRAP, TRAP_HWBKPT,
- "breakpoint debug exception");
+ hook_fault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP,
+ TRAP_HWBKPT, "watchpoint debug exception");
+ hook_ifault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP,
+ TRAP_HWBKPT, "breakpoint debug exception");
/* Register hotplug notifier. */
register_cpu_notifier(&dbg_reset_nb);
.pushsection .idmap.text,"ax"
ENTRY(cpu_resume_mmu)
ldr r3, =cpu_resume_after_mmu
+ instr_sync
mcr p15, 0, r0, c1, c0, 0 @ turn on MMU, I-cache, etc
mrc p15, 0, r0, c0, c0, 0 @ read id reg
+ instr_sync
mov r0, r0
mov r0, r0
mov pc, r3 @ jump to virtual address
comment "Processor Features"
+config ARM_LPAE
+ bool "Support for the Large Physical Address Extension"
+ depends on MMU && CPU_V7
+ help
+ Say Y if you have an ARMv7 processor supporting the LPAE page
+ table format and you would like to access memory beyond the
+ 4GB limit. The resulting kernel image will not run on
+ processors without the LPA extension.
+
+ If unsure, say N.
+
+config ARCH_PHYS_ADDR_T_64BIT
+ def_bool ARM_LPAE
+
+config ARCH_DMA_ADDR_T_64BIT
+ bool
+
config ARM_THUMB
bool "Support Thumb user binaries"
depends on CPU_ARM720T || CPU_ARM740T || CPU_ARM920T || CPU_ARM922T || CPU_ARM925T || CPU_ARM926T || CPU_ARM940T || CPU_ARM946E || CPU_ARM1020 || CPU_ARM1020E || CPU_ARM1022 || CPU_ARM1026 || CPU_XSCALE || CPU_XSC3 || CPU_MOHAWK || CPU_V6 || CPU_V6K || CPU_V7 || CPU_FEROCEON
ai_usermode = safe_usermode(ai_usermode, false);
}
- hook_fault_code(1, do_alignment, SIGBUS, BUS_ADRALN,
+ hook_fault_code(FAULT_CODE_ALIGNMENT, do_alignment, SIGBUS, BUS_ADRALN,
"alignment exception");
/*
DEFINE_PER_CPU(struct mm_struct *, current_mm);
#endif
+#ifdef CONFIG_ARM_LPAE
+#define cpu_set_asid(asid) { \
+ unsigned long ttbl, ttbh; \
+ asm volatile( \
+ " mrrc p15, 0, %0, %1, c2 @ read TTBR0\n" \
+ " mov %1, %2, lsl #(48 - 32) @ set ASID\n" \
+ " mcrr p15, 0, %0, %1, c2 @ set TTBR0\n" \
+ : "=&r" (ttbl), "=&r" (ttbh) \
+ : "r" (asid & ~ASID_MASK)); \
+}
+#else
+#define cpu_set_asid(asid) \
+ asm(" mcr p15, 0, %0, c13, c0, 1\n" : : "r" (asid))
+#endif
+
/*
* We fork()ed a process, and we need a new context for the child
* to run in. We reserve version 0 for initial tasks so we will
static void flush_context(void)
{
/* set the reserved ASID before flushing the TLB */
- asm("mcr p15, 0, %0, c13, c0, 1\n" : : "r" (0));
+ cpu_set_asid(0);
isb();
local_flush_tlb_all();
if (icache_is_vivt_asid_tagged()) {
set_mm_context(mm, asid);
/* set the new ASID */
- asm("mcr p15, 0, %0, c13, c0, 1\n" : : "r" (mm->context.id));
+ cpu_set_asid(mm->context.id);
isb();
}
#include "fault.h"
-/*
- * Fault status register encodings. We steal bit 31 for our own purposes.
- */
-#define FSR_LNX_PF (1 << 31)
-#define FSR_WRITE (1 << 11)
-#define FSR_FS4 (1 << 10)
-#define FSR_FS3_0 (15)
-
-static inline int fsr_fs(unsigned int fsr)
-{
- return (fsr & FSR_FS3_0) | (fsr & FSR_FS4) >> 6;
-}
-
#ifdef CONFIG_MMU
#ifdef CONFIG_KPROBES
pte = pte_offset_map(pmd, addr);
printk(", *pte=%08llx", (long long)pte_val(*pte));
+#ifndef CONFIG_ARM_LPAE
printk(", *ppte=%08llx",
(long long)pte_val(pte[PTE_HWTABLE_PTRS]));
+#endif
pte_unmap(pte);
} while(0);
pmd = pmd_offset(pud, addr);
pmd_k = pmd_offset(pud_k, addr);
+#ifdef CONFIG_ARM_LPAE
+ /*
+ * Only one hardware entry per PMD with LPAE.
+ */
+ index = 0;
+#else
/*
* On ARM one Linux PGD entry contains two hardware entries (see page
* tables layout in pgtable.h). We normally guarantee that we always
* for the first of pair.
*/
index = (addr >> SECTION_SHIFT) & 1;
+#endif
if (pmd_none(pmd_k[index]))
goto bad_area;
return 1;
}
-static struct fsr_info {
+struct fsr_info {
int (*fn)(unsigned long addr, unsigned int fsr, struct pt_regs *regs);
int sig;
int code;
const char *name;
-} fsr_info[] = {
- /*
- * The following are the standard ARMv3 and ARMv4 aborts. ARMv5
- * defines these to be "precise" aborts.
- */
- { do_bad, SIGSEGV, 0, "vector exception" },
- { do_bad, SIGBUS, BUS_ADRALN, "alignment exception" },
- { do_bad, SIGKILL, 0, "terminal exception" },
- { do_bad, SIGBUS, BUS_ADRALN, "alignment exception" },
- { do_bad, SIGBUS, 0, "external abort on linefetch" },
- { do_translation_fault, SIGSEGV, SEGV_MAPERR, "section translation fault" },
- { do_bad, SIGBUS, 0, "external abort on linefetch" },
- { do_page_fault, SIGSEGV, SEGV_MAPERR, "page translation fault" },
- { do_bad, SIGBUS, 0, "external abort on non-linefetch" },
- { do_bad, SIGSEGV, SEGV_ACCERR, "section domain fault" },
- { do_bad, SIGBUS, 0, "external abort on non-linefetch" },
- { do_bad, SIGSEGV, SEGV_ACCERR, "page domain fault" },
- { do_bad, SIGBUS, 0, "external abort on translation" },
- { do_sect_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" },
- { do_bad, SIGBUS, 0, "external abort on translation" },
- { do_page_fault, SIGSEGV, SEGV_ACCERR, "page permission fault" },
- /*
- * The following are "imprecise" aborts, which are signalled by bit
- * 10 of the FSR, and may not be recoverable. These are only
- * supported if the CPU abort handler supports bit 10.
- */
- { do_bad, SIGBUS, 0, "unknown 16" },
- { do_bad, SIGBUS, 0, "unknown 17" },
- { do_bad, SIGBUS, 0, "unknown 18" },
- { do_bad, SIGBUS, 0, "unknown 19" },
- { do_bad, SIGBUS, 0, "lock abort" }, /* xscale */
- { do_bad, SIGBUS, 0, "unknown 21" },
- { do_bad, SIGBUS, BUS_OBJERR, "imprecise external abort" }, /* xscale */
- { do_bad, SIGBUS, 0, "unknown 23" },
- { do_bad, SIGBUS, 0, "dcache parity error" }, /* xscale */
- { do_bad, SIGBUS, 0, "unknown 25" },
- { do_bad, SIGBUS, 0, "unknown 26" },
- { do_bad, SIGBUS, 0, "unknown 27" },
- { do_bad, SIGBUS, 0, "unknown 28" },
- { do_bad, SIGBUS, 0, "unknown 29" },
- { do_bad, SIGBUS, 0, "unknown 30" },
- { do_bad, SIGBUS, 0, "unknown 31" }
};
+/* FSR definition */
+#ifdef CONFIG_ARM_LPAE
+#include "fsr-3level.c"
+#else
+#include "fsr-2level.c"
+#endif
+
void __init
hook_fault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *),
int sig, int code, const char *name)
arm_notify_die("", regs, &info, fsr, 0);
}
-
-static struct fsr_info ifsr_info[] = {
- { do_bad, SIGBUS, 0, "unknown 0" },
- { do_bad, SIGBUS, 0, "unknown 1" },
- { do_bad, SIGBUS, 0, "debug event" },
- { do_bad, SIGSEGV, SEGV_ACCERR, "section access flag fault" },
- { do_bad, SIGBUS, 0, "unknown 4" },
- { do_translation_fault, SIGSEGV, SEGV_MAPERR, "section translation fault" },
- { do_bad, SIGSEGV, SEGV_ACCERR, "page access flag fault" },
- { do_page_fault, SIGSEGV, SEGV_MAPERR, "page translation fault" },
- { do_bad, SIGBUS, 0, "external abort on non-linefetch" },
- { do_bad, SIGSEGV, SEGV_ACCERR, "section domain fault" },
- { do_bad, SIGBUS, 0, "unknown 10" },
- { do_bad, SIGSEGV, SEGV_ACCERR, "page domain fault" },
- { do_bad, SIGBUS, 0, "external abort on translation" },
- { do_sect_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" },
- { do_bad, SIGBUS, 0, "external abort on translation" },
- { do_page_fault, SIGSEGV, SEGV_ACCERR, "page permission fault" },
- { do_bad, SIGBUS, 0, "unknown 16" },
- { do_bad, SIGBUS, 0, "unknown 17" },
- { do_bad, SIGBUS, 0, "unknown 18" },
- { do_bad, SIGBUS, 0, "unknown 19" },
- { do_bad, SIGBUS, 0, "unknown 20" },
- { do_bad, SIGBUS, 0, "unknown 21" },
- { do_bad, SIGBUS, 0, "unknown 22" },
- { do_bad, SIGBUS, 0, "unknown 23" },
- { do_bad, SIGBUS, 0, "unknown 24" },
- { do_bad, SIGBUS, 0, "unknown 25" },
- { do_bad, SIGBUS, 0, "unknown 26" },
- { do_bad, SIGBUS, 0, "unknown 27" },
- { do_bad, SIGBUS, 0, "unknown 28" },
- { do_bad, SIGBUS, 0, "unknown 29" },
- { do_bad, SIGBUS, 0, "unknown 30" },
- { do_bad, SIGBUS, 0, "unknown 31" },
-};
-
void __init
hook_ifault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *),
int sig, int code, const char *name)
arm_notify_die("", regs, &info, ifsr, 0);
}
+#ifndef CONFIG_ARM_LPAE
static int __init exceptions_init(void)
{
if (cpu_architecture() >= CPU_ARCH_ARMv6) {
}
arch_initcall(exceptions_init);
+#endif
-void do_bad_area(unsigned long addr, unsigned int fsr, struct pt_regs *regs);
+#ifndef __ARCH_ARM_FAULT_H
+#define __ARCH_ARM_FAULT_H
+
+/*
+ * Fault status register encodings. We steal bit 31 for our own purposes.
+ */
+#define FSR_LNX_PF (1 << 31)
+#define FSR_WRITE (1 << 11)
+#define FSR_FS4 (1 << 10)
+#define FSR_FS3_0 (15)
+#define FSR_FS5_0 (0x3f)
+
+#ifdef CONFIG_ARM_LPAE
+static inline int fsr_fs(unsigned int fsr)
+{
+ return fsr & FSR_FS5_0;
+}
+#else
+static inline int fsr_fs(unsigned int fsr)
+{
+ return (fsr & FSR_FS3_0) | (fsr & FSR_FS4) >> 6;
+}
+#endif
+void do_bad_area(unsigned long addr, unsigned int fsr, struct pt_regs *regs);
unsigned long search_exception_table(unsigned long addr);
+
+#endif /* __ARCH_ARM_FAULT_H */
--- /dev/null
+static struct fsr_info fsr_info[] = {
+ /*
+ * The following are the standard ARMv3 and ARMv4 aborts. ARMv5
+ * defines these to be "precise" aborts.
+ */
+ { do_bad, SIGSEGV, 0, "vector exception" },
+ { do_bad, SIGBUS, BUS_ADRALN, "alignment exception" },
+ { do_bad, SIGKILL, 0, "terminal exception" },
+ { do_bad, SIGBUS, BUS_ADRALN, "alignment exception" },
+ { do_bad, SIGBUS, 0, "external abort on linefetch" },
+ { do_translation_fault, SIGSEGV, SEGV_MAPERR, "section translation fault" },
+ { do_bad, SIGBUS, 0, "external abort on linefetch" },
+ { do_page_fault, SIGSEGV, SEGV_MAPERR, "page translation fault" },
+ { do_bad, SIGBUS, 0, "external abort on non-linefetch" },
+ { do_bad, SIGSEGV, SEGV_ACCERR, "section domain fault" },
+ { do_bad, SIGBUS, 0, "external abort on non-linefetch" },
+ { do_bad, SIGSEGV, SEGV_ACCERR, "page domain fault" },
+ { do_bad, SIGBUS, 0, "external abort on translation" },
+ { do_sect_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" },
+ { do_bad, SIGBUS, 0, "external abort on translation" },
+ { do_page_fault, SIGSEGV, SEGV_ACCERR, "page permission fault" },
+ /*
+ * The following are "imprecise" aborts, which are signalled by bit
+ * 10 of the FSR, and may not be recoverable. These are only
+ * supported if the CPU abort handler supports bit 10.
+ */
+ { do_bad, SIGBUS, 0, "unknown 16" },
+ { do_bad, SIGBUS, 0, "unknown 17" },
+ { do_bad, SIGBUS, 0, "unknown 18" },
+ { do_bad, SIGBUS, 0, "unknown 19" },
+ { do_bad, SIGBUS, 0, "lock abort" }, /* xscale */
+ { do_bad, SIGBUS, 0, "unknown 21" },
+ { do_bad, SIGBUS, BUS_OBJERR, "imprecise external abort" }, /* xscale */
+ { do_bad, SIGBUS, 0, "unknown 23" },
+ { do_bad, SIGBUS, 0, "dcache parity error" }, /* xscale */
+ { do_bad, SIGBUS, 0, "unknown 25" },
+ { do_bad, SIGBUS, 0, "unknown 26" },
+ { do_bad, SIGBUS, 0, "unknown 27" },
+ { do_bad, SIGBUS, 0, "unknown 28" },
+ { do_bad, SIGBUS, 0, "unknown 29" },
+ { do_bad, SIGBUS, 0, "unknown 30" },
+ { do_bad, SIGBUS, 0, "unknown 31" },
+};
+
+static struct fsr_info ifsr_info[] = {
+ { do_bad, SIGBUS, 0, "unknown 0" },
+ { do_bad, SIGBUS, 0, "unknown 1" },
+ { do_bad, SIGBUS, 0, "debug event" },
+ { do_bad, SIGSEGV, SEGV_ACCERR, "section access flag fault" },
+ { do_bad, SIGBUS, 0, "unknown 4" },
+ { do_translation_fault, SIGSEGV, SEGV_MAPERR, "section translation fault" },
+ { do_bad, SIGSEGV, SEGV_ACCERR, "page access flag fault" },
+ { do_page_fault, SIGSEGV, SEGV_MAPERR, "page translation fault" },
+ { do_bad, SIGBUS, 0, "external abort on non-linefetch" },
+ { do_bad, SIGSEGV, SEGV_ACCERR, "section domain fault" },
+ { do_bad, SIGBUS, 0, "unknown 10" },
+ { do_bad, SIGSEGV, SEGV_ACCERR, "page domain fault" },
+ { do_bad, SIGBUS, 0, "external abort on translation" },
+ { do_sect_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" },
+ { do_bad, SIGBUS, 0, "external abort on translation" },
+ { do_page_fault, SIGSEGV, SEGV_ACCERR, "page permission fault" },
+ { do_bad, SIGBUS, 0, "unknown 16" },
+ { do_bad, SIGBUS, 0, "unknown 17" },
+ { do_bad, SIGBUS, 0, "unknown 18" },
+ { do_bad, SIGBUS, 0, "unknown 19" },
+ { do_bad, SIGBUS, 0, "unknown 20" },
+ { do_bad, SIGBUS, 0, "unknown 21" },
+ { do_bad, SIGBUS, 0, "unknown 22" },
+ { do_bad, SIGBUS, 0, "unknown 23" },
+ { do_bad, SIGBUS, 0, "unknown 24" },
+ { do_bad, SIGBUS, 0, "unknown 25" },
+ { do_bad, SIGBUS, 0, "unknown 26" },
+ { do_bad, SIGBUS, 0, "unknown 27" },
+ { do_bad, SIGBUS, 0, "unknown 28" },
+ { do_bad, SIGBUS, 0, "unknown 29" },
+ { do_bad, SIGBUS, 0, "unknown 30" },
+ { do_bad, SIGBUS, 0, "unknown 31" },
+};
--- /dev/null
+static struct fsr_info fsr_info[] = {
+ { do_bad, SIGBUS, 0, "unknown 0" },
+ { do_bad, SIGBUS, 0, "unknown 1" },
+ { do_bad, SIGBUS, 0, "unknown 2" },
+ { do_bad, SIGBUS, 0, "unknown 3" },
+ { do_bad, SIGBUS, 0, "reserved translation fault" },
+ { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 1 translation fault" },
+ { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 2 translation fault" },
+ { do_page_fault, SIGSEGV, SEGV_MAPERR, "level 3 translation fault" },
+ { do_bad, SIGBUS, 0, "reserved access flag fault" },
+ { do_bad, SIGSEGV, SEGV_ACCERR, "level 1 access flag fault" },
+ { do_bad, SIGSEGV, SEGV_ACCERR, "level 2 access flag fault" },
+ { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 access flag fault" },
+ { do_bad, SIGBUS, 0, "reserved permission fault" },
+ { do_bad, SIGSEGV, SEGV_ACCERR, "level 1 permission fault" },
+ { do_sect_fault, SIGSEGV, SEGV_ACCERR, "level 2 permission fault" },
+ { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 permission fault" },
+ { do_bad, SIGBUS, 0, "synchronous external abort" },
+ { do_bad, SIGBUS, 0, "asynchronous external abort" },
+ { do_bad, SIGBUS, 0, "unknown 18" },
+ { do_bad, SIGBUS, 0, "unknown 19" },
+ { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" },
+ { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" },
+ { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" },
+ { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" },
+ { do_bad, SIGBUS, 0, "synchronous parity error" },
+ { do_bad, SIGBUS, 0, "asynchronous parity error" },
+ { do_bad, SIGBUS, 0, "unknown 26" },
+ { do_bad, SIGBUS, 0, "unknown 27" },
+ { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk" },
+ { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk" },
+ { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk" },
+ { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk" },
+ { do_bad, SIGBUS, 0, "unknown 32" },
+ { do_bad, SIGBUS, BUS_ADRALN, "alignment fault" },
+ { do_bad, SIGBUS, 0, "debug event" },
+ { do_bad, SIGBUS, 0, "unknown 35" },
+ { do_bad, SIGBUS, 0, "unknown 36" },
+ { do_bad, SIGBUS, 0, "unknown 37" },
+ { do_bad, SIGBUS, 0, "unknown 38" },
+ { do_bad, SIGBUS, 0, "unknown 39" },
+ { do_bad, SIGBUS, 0, "unknown 40" },
+ { do_bad, SIGBUS, 0, "unknown 41" },
+ { do_bad, SIGBUS, 0, "unknown 42" },
+ { do_bad, SIGBUS, 0, "unknown 43" },
+ { do_bad, SIGBUS, 0, "unknown 44" },
+ { do_bad, SIGBUS, 0, "unknown 45" },
+ { do_bad, SIGBUS, 0, "unknown 46" },
+ { do_bad, SIGBUS, 0, "unknown 47" },
+ { do_bad, SIGBUS, 0, "unknown 48" },
+ { do_bad, SIGBUS, 0, "unknown 49" },
+ { do_bad, SIGBUS, 0, "unknown 50" },
+ { do_bad, SIGBUS, 0, "unknown 51" },
+ { do_bad, SIGBUS, 0, "implementation fault (lockdown abort)" },
+ { do_bad, SIGBUS, 0, "unknown 53" },
+ { do_bad, SIGBUS, 0, "unknown 54" },
+ { do_bad, SIGBUS, 0, "unknown 55" },
+ { do_bad, SIGBUS, 0, "unknown 56" },
+ { do_bad, SIGBUS, 0, "unknown 57" },
+ { do_bad, SIGBUS, 0, "implementation fault (coprocessor abort)" },
+ { do_bad, SIGBUS, 0, "unknown 59" },
+ { do_bad, SIGBUS, 0, "unknown 60" },
+ { do_bad, SIGBUS, 0, "unknown 61" },
+ { do_bad, SIGBUS, 0, "unknown 62" },
+ { do_bad, SIGBUS, 0, "unknown 63" },
+};
+
+#define ifsr_info fsr_info
pgd_t *idmap_pgd;
+#ifdef CONFIG_ARM_LPAE
+static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end,
+ unsigned long prot)
+{
+ pmd_t *pmd;
+ unsigned long next;
+
+ if (pud_none_or_clear_bad(pud) || (pud_val(*pud) & L_PGD_SWAPPER)) {
+ pmd = pmd_alloc_one(&init_mm, addr);
+ if (!pmd) {
+ pr_warning("Failed to allocate identity pmd.\n");
+ return;
+ }
+ pud_populate(&init_mm, pud, pmd);
+ pmd += pmd_index(addr);
+ } else
+ pmd = pmd_offset(pud, addr);
+
+ do {
+ next = pmd_addr_end(addr, end);
+ *pmd = __pmd((addr & PMD_MASK) | prot);
+ flush_pmd_entry(pmd);
+ } while (pmd++, addr = next, addr != end);
+}
+#else /* !CONFIG_ARM_LPAE */
static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end,
unsigned long prot)
{
pmd[1] = __pmd(addr);
flush_pmd_entry(pmd);
}
+#endif /* CONFIG_ARM_LPAE */
static void idmap_add_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
unsigned long prot)
{
unsigned long prot, next;
- prot = PMD_TYPE_SECT | PMD_SECT_AP_WRITE;
+ prot = PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AF;
if (cpu_architecture() <= CPU_ARCH_ARMv5TEJ && !cpu_is_xscale())
prot |= PMD_BIT4;
} while (seq != init_mm.context.kvm_seq);
}
-#ifndef CONFIG_SMP
+#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
/*
* Section support is unsafe on SMP - If you iounmap and ioremap a region,
* the other CPUs will not see this change until their next context switch.
{
unsigned long addr = virt, end = virt + (size & ~(SZ_1M - 1));
pgd_t *pgd;
+ pud_t *pud;
+ pmd_t *pmdp;
flush_cache_vunmap(addr, end);
pgd = pgd_offset_k(addr);
+ pud = pud_offset(pgd, addr);
+ pmdp = pmd_offset(pud, addr);
do {
- pmd_t pmd, *pmdp = pmd_offset(pgd, addr);
+ pmd_t pmd = *pmdp;
- pmd = *pmdp;
if (!pmd_none(pmd)) {
/*
* Clear the PMD from the page table, and
pte_free_kernel(&init_mm, pmd_page_vaddr(pmd));
}
- addr += PGDIR_SIZE;
- pgd++;
+ addr += PMD_SIZE;
+ pmdp += 2;
} while (addr < end);
/*
{
unsigned long addr = virt, end = virt + size;
pgd_t *pgd;
+ pud_t *pud;
+ pmd_t *pmd;
/*
* Remove and free any PTE-based mapping, and
unmap_area_sections(virt, size);
pgd = pgd_offset_k(addr);
+ pud = pud_offset(pgd, addr);
+ pmd = pmd_offset(pud, addr);
do {
- pmd_t *pmd = pmd_offset(pgd, addr);
-
pmd[0] = __pmd(__pfn_to_phys(pfn) | type->prot_sect);
pfn += SZ_1M >> PAGE_SHIFT;
pmd[1] = __pmd(__pfn_to_phys(pfn) | type->prot_sect);
pfn += SZ_1M >> PAGE_SHIFT;
flush_pmd_entry(pmd);
- addr += PGDIR_SIZE;
- pgd++;
+ addr += PMD_SIZE;
+ pmd += 2;
} while (addr < end);
return 0;
{
unsigned long addr = virt, end = virt + size;
pgd_t *pgd;
+ pud_t *pud;
+ pmd_t *pmd;
/*
* Remove and free any PTE-based mapping, and
unmap_area_sections(virt, size);
pgd = pgd_offset_k(virt);
+ pud = pud_offset(pgd, addr);
+ pmd = pmd_offset(pud, addr);
do {
unsigned long super_pmd_val, i;
super_pmd_val |= ((pfn >> (32 - PAGE_SHIFT)) & 0xf) << 20;
for (i = 0; i < 8; i++) {
- pmd_t *pmd = pmd_offset(pgd, addr);
-
pmd[0] = __pmd(super_pmd_val);
pmd[1] = __pmd(super_pmd_val);
flush_pmd_entry(pmd);
- addr += PGDIR_SIZE;
- pgd++;
+ addr += PMD_SIZE;
+ pmd += 2;
}
pfn += SUPERSECTION_SIZE >> PAGE_SHIFT;
unsigned long addr;
struct vm_struct * area;
+#ifndef CONFIG_ARM_LPAE
/*
* High mappings must be supersection aligned
*/
if (pfn >= 0x100000 && (__pfn_to_phys(pfn) & ~SUPERSECTION_MASK))
return NULL;
+#endif
type = get_mem_type(mtype);
if (!type)
return NULL;
addr = (unsigned long)area->addr;
-#ifndef CONFIG_SMP
+#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
if (DOMAIN_IO == 0 &&
(((cpu_architecture() >= CPU_ARCH_ARMv6) && (get_cr() & CR_XP)) ||
cpu_is_xsc3()) && pfn >= 0x100000 &&
read_unlock(&vmlist_lock);
return;
}
-#ifndef CONFIG_SMP
+#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
/*
* If this is a section based mapping we need to handle it
* specially as the VM subsystem does not know how to handle
}
early_param("nowb", early_nowrite);
+#ifndef CONFIG_ARM_LPAE
static int __init early_ecc(char *p)
{
if (memcmp(p, "on", 2) == 0)
return 0;
}
early_param("ecc", early_ecc);
+#endif
static int __init noalign_setup(char *__unused)
{
.prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
.domain = DOMAIN_KERNEL,
},
+#ifndef CONFIG_ARM_LPAE
[MT_MINICLEAN] = {
.prot_sect = PMD_TYPE_SECT | PMD_SECT_XN | PMD_SECT_MINICACHE,
.domain = DOMAIN_KERNEL,
},
+#endif
[MT_LOW_VECTORS] = {
.prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
L_PTE_RDONLY,
* ARMv6 and above have extended page tables.
*/
if (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP)) {
+#ifndef CONFIG_ARM_LPAE
/*
* Mark cache clean areas and XIP ROM read only
* from SVC mode and no access from userspace.
mem_types[MT_ROM].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
mem_types[MT_MINICLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
+#endif
if (is_smp()) {
/*
mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_BUFFERABLE;
}
+#ifdef CONFIG_ARM_LPAE
+ /*
+ * Do not generate access flag faults for the kernel mappings.
+ */
+ for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
+ mem_types[i].prot_pte |= PTE_EXT_AF;
+ mem_types[i].prot_sect |= PMD_SECT_AF;
+ }
+ kern_pgprot |= PTE_EXT_AF;
+ vecs_pgprot |= PTE_EXT_AF;
+#endif
+
for (i = 0; i < 16; i++) {
unsigned long v = pgprot_val(protection_map[i]);
protection_map[i] = __pgprot(v | user_pgprot);
if (((addr | end | phys) & ~SECTION_MASK) == 0) {
pmd_t *p = pmd;
+#ifndef CONFIG_ARM_LPAE
if (addr & SECTION_SIZE)
pmd++;
+#endif
do {
*pmd = __pmd(phys | type->prot_sect);
} while (pud++, addr = next, addr != end);
}
+#ifndef CONFIG_ARM_LPAE
static void __init create_36bit_mapping(struct map_desc *md,
const struct mem_type *type)
{
pgd += SUPERSECTION_SIZE >> PGDIR_SHIFT;
} while (addr != end);
}
+#endif /* !CONFIG_ARM_LPAE */
/*
* Create the page directory entries and any necessary
type = &mem_types[md->type];
+#ifndef CONFIG_ARM_LPAE
/*
* Catch 36-bit addresses
*/
create_36bit_mapping(md, type);
return;
}
+#endif
addr = md->virtual & PAGE_MASK;
phys = __pfn_to_phys(md->pfn);
struct membank *bank = &meminfo.bank[j];
*bank = meminfo.bank[i];
+ if (bank->start > ULONG_MAX)
+ highmem = 1;
+
#ifdef CONFIG_HIGHMEM
if (__va(bank->start) >= vmalloc_min ||
__va(bank->start) < (void *)PAGE_OFFSET)
* Split those memory banks which are partially overlapping
* the vmalloc area greatly simplifying things later.
*/
- if (__va(bank->start) < vmalloc_min &&
+ if (!highmem && __va(bank->start) < vmalloc_min &&
bank->size > vmalloc_min - __va(bank->start)) {
if (meminfo.nr_banks >= NR_BANKS) {
printk(KERN_CRIT "NR_BANKS too low, "
#else
bank->highmem = highmem;
+ /*
+ * Highmem banks not allowed with !CONFIG_HIGHMEM.
+ */
+ if (highmem) {
+ printk(KERN_NOTICE "Ignoring RAM at %.8llx-%.8llx "
+ "(!CONFIG_HIGHMEM).\n",
+ (unsigned long long)bank->start,
+ (unsigned long long)bank->start + bank->size - 1);
+ continue;
+ }
+
/*
* Check whether this memory bank would entirely overlap
* the vmalloc area.
pmd_clear(pmd_off_k(addr));
}
+#ifdef CONFIG_ARM_LPAE
+/* the first page is reserved for pgd */
+#define SWAPPER_PG_DIR_SIZE (PAGE_SIZE + \
+ PTRS_PER_PGD * PTRS_PER_PMD * sizeof(pmd_t))
+#else
#define SWAPPER_PG_DIR_SIZE (PTRS_PER_PGD * sizeof(pgd_t))
+#endif
/*
* Reserve the special regions of memory
#include <linux/mm.h>
#include <linux/gfp.h>
#include <linux/highmem.h>
+#include <linux/slab.h>
#include <asm/pgalloc.h>
#include <asm/page.h>
#include "mm.h"
+#ifdef CONFIG_ARM_LPAE
+#define __pgd_alloc() kmalloc(PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL)
+#define __pgd_free(pgd) kfree(pgd)
+#else
+#define __pgd_alloc() (pgd_t *)__get_free_pages(GFP_KERNEL, 2)
+#define __pgd_free(pgd) free_pages((unsigned long)pgd, 2)
+#endif
+
/*
* need to get a 16k page for level 1
*/
pmd_t *new_pmd, *init_pmd;
pte_t *new_pte, *init_pte;
- new_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL, 2);
+ new_pgd = __pgd_alloc();
if (!new_pgd)
goto no_pgd;
clean_dcache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t));
+#ifdef CONFIG_ARM_LPAE
+ /*
+ * Allocate PMD table for modules and pkmap mappings.
+ */
+ new_pud = pud_alloc(mm, new_pgd + pgd_index(MODULES_VADDR),
+ MODULES_VADDR);
+ if (!new_pud)
+ goto no_pud;
+
+ new_pmd = pmd_alloc(mm, new_pud, 0);
+ if (!new_pmd)
+ goto no_pmd;
+#endif
+
if (!vectors_high()) {
/*
* On ARM, first page must always be allocated since it
- * contains the machine vectors.
+ * contains the machine vectors. The vectors are always high
+ * with LPAE.
*/
new_pud = pud_alloc(mm, new_pgd, 0);
if (!new_pud)
no_pmd:
pud_free(mm, new_pud);
no_pud:
- free_pages((unsigned long)new_pgd, 2);
+ __pgd_free(new_pgd);
no_pgd:
return NULL;
}
pgd_clear(pgd);
pud_free(mm, pud);
no_pgd:
- free_pages((unsigned long) pgd_base, 2);
+#ifdef CONFIG_ARM_LPAE
+ /*
+ * Free modules/pkmap or identity pmd tables.
+ */
+ for (pgd = pgd_base; pgd < pgd_base + PTRS_PER_PGD; pgd++) {
+ if (pgd_none_or_clear_bad(pgd))
+ continue;
+ if (pgd_val(*pgd) & L_PGD_SWAPPER)
+ continue;
+ pud = pud_offset(pgd, 0);
+ if (pud_none_or_clear_bad(pud))
+ continue;
+ pmd = pmd_offset(pud, 0);
+ pud_clear(pud);
+ pmd_free(mm, pmd);
+ pgd_clear(pgd);
+ pud_free(mm, pud);
+ }
+#endif
+ __pgd_free(pgd_base);
}
#if L_PTE_SHARED != PTE_EXT_SHARED
#error PTE shared bit mismatch
#endif
-#if (L_PTE_XN+L_PTE_USER+L_PTE_RDONLY+L_PTE_DIRTY+L_PTE_YOUNG+\
- L_PTE_FILE+L_PTE_PRESENT) > L_PTE_SHARED
+#if !defined (CONFIG_ARM_LPAE) && \
+ (L_PTE_XN+L_PTE_USER+L_PTE_RDONLY+L_PTE_DIRTY+L_PTE_YOUNG+\
+ L_PTE_FILE+L_PTE_PRESENT) > L_PTE_SHARED
#error Invalid Linux PTE bit settings
#endif
#endif /* CONFIG_MMU */
--- /dev/null
+/*
+ * arch/arm/mm/proc-v7-2level.S
+ *
+ * Copyright (C) 2001 Deep Blue Solutions Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#define TTB_S (1 << 1)
+#define TTB_RGN_NC (0 << 3)
+#define TTB_RGN_OC_WBWA (1 << 3)
+#define TTB_RGN_OC_WT (2 << 3)
+#define TTB_RGN_OC_WB (3 << 3)
+#define TTB_NOS (1 << 5)
+#define TTB_IRGN_NC ((0 << 0) | (0 << 6))
+#define TTB_IRGN_WBWA ((0 << 0) | (1 << 6))
+#define TTB_IRGN_WT ((1 << 0) | (0 << 6))
+#define TTB_IRGN_WB ((1 << 0) | (1 << 6))
+
+/* PTWs cacheable, inner WB not shareable, outer WB not shareable */
+#define TTB_FLAGS_UP TTB_IRGN_WB|TTB_RGN_OC_WB
+#define PMD_FLAGS_UP PMD_SECT_WB
+
+/* PTWs cacheable, inner WBWA shareable, outer WBWA not shareable */
+#define TTB_FLAGS_SMP TTB_IRGN_WBWA|TTB_S|TTB_NOS|TTB_RGN_OC_WBWA
+#define PMD_FLAGS_SMP PMD_SECT_WBWA|PMD_SECT_S
+
+/*
+ * cpu_v7_switch_mm(pgd_phys, tsk)
+ *
+ * Set the translation table base pointer to be pgd_phys
+ *
+ * - pgd_phys - physical address of new TTB
+ *
+ * It is assumed that:
+ * - we are not using split page tables
+ */
+ENTRY(cpu_v7_switch_mm)
+#ifdef CONFIG_MMU
+ mov r2, #0
+ ldr r1, [r1, #MM_CONTEXT_ID] @ get mm->context.id
+ ALT_SMP(orr r0, r0, #TTB_FLAGS_SMP)
+ ALT_UP(orr r0, r0, #TTB_FLAGS_UP)
+#ifdef CONFIG_ARM_ERRATA_430973
+ mcr p15, 0, r2, c7, c5, 6 @ flush BTAC/BTB
+#endif
+#ifdef CONFIG_ARM_ERRATA_754322
+ dsb
+#endif
+ mcr p15, 0, r2, c13, c0, 1 @ set reserved context ID
+ isb
+1: mcr p15, 0, r0, c2, c0, 0 @ set TTB 0
+ isb
+#ifdef CONFIG_ARM_ERRATA_754322
+ dsb
+#endif
+ mcr p15, 0, r1, c13, c0, 1 @ set context ID
+ isb
+#endif
+ mov pc, lr
+ENDPROC(cpu_v7_switch_mm)
+
+/*
+ * cpu_v7_set_pte_ext(ptep, pte)
+ *
+ * Set a level 2 translation table entry.
+ *
+ * - ptep - pointer to level 2 translation table entry
+ * (hardware version is stored at +2048 bytes)
+ * - pte - PTE value to store
+ * - ext - value for extended PTE bits
+ */
+ENTRY(cpu_v7_set_pte_ext)
+#ifdef CONFIG_MMU
+ str r1, [r0] @ linux version
+
+ bic r3, r1, #0x000003f0
+ bic r3, r3, #PTE_TYPE_MASK
+ orr r3, r3, r2
+ orr r3, r3, #PTE_EXT_AP0 | 2
+
+ tst r1, #1 << 4
+ orrne r3, r3, #PTE_EXT_TEX(1)
+
+ eor r1, r1, #L_PTE_DIRTY
+ tst r1, #L_PTE_RDONLY | L_PTE_DIRTY
+ orrne r3, r3, #PTE_EXT_APX
+
+ tst r1, #L_PTE_USER
+ orrne r3, r3, #PTE_EXT_AP1
+#ifdef CONFIG_CPU_USE_DOMAINS
+ @ allow kernel read/write access to read-only user pages
+ tstne r3, #PTE_EXT_APX
+ bicne r3, r3, #PTE_EXT_APX | PTE_EXT_AP0
+#endif
+
+ tst r1, #L_PTE_XN
+ orrne r3, r3, #PTE_EXT_XN
+
+ tst r1, #L_PTE_YOUNG
+ tstne r1, #L_PTE_PRESENT
+ moveq r3, #0
+
+ ARM( str r3, [r0, #2048]! )
+ THUMB( add r0, r0, #2048 )
+ THUMB( str r3, [r0] )
+ mcr p15, 0, r0, c7, c10, 1 @ flush_pte
+#endif
+ mov pc, lr
+ENDPROC(cpu_v7_set_pte_ext)
+
+ /*
+ * Memory region attributes with SCTLR.TRE=1
+ *
+ * n = TEX[0],C,B
+ * TR = PRRR[2n+1:2n] - memory type
+ * IR = NMRR[2n+1:2n] - inner cacheable property
+ * OR = NMRR[2n+17:2n+16] - outer cacheable property
+ *
+ * n TR IR OR
+ * UNCACHED 000 00
+ * BUFFERABLE 001 10 00 00
+ * WRITETHROUGH 010 10 10 10
+ * WRITEBACK 011 10 11 11
+ * reserved 110
+ * WRITEALLOC 111 10 01 01
+ * DEV_SHARED 100 01
+ * DEV_NONSHARED 100 01
+ * DEV_WC 001 10
+ * DEV_CACHED 011 10
+ *
+ * Other attributes:
+ *
+ * DS0 = PRRR[16] = 0 - device shareable property
+ * DS1 = PRRR[17] = 1 - device shareable property
+ * NS0 = PRRR[18] = 0 - normal shareable property
+ * NS1 = PRRR[19] = 1 - normal shareable property
+ * NOS = PRRR[24+n] = 1 - not outer shareable
+ */
+.equ PRRR, 0xff0a81a8
+.equ NMRR, 0x40e040e0
+
+ /*
+ * Macro for setting up the TTBRx and TTBCR registers.
+ * - \ttb0 and \ttb1 updated with the corresponding flags.
+ */
+ .macro v7_ttb_setup, zero, ttbr0, ttbr1, tmp
+ mcr p15, 0, \zero, c2, c0, 2 @ TTB control register
+ ALT_SMP(orr \ttbr0, \ttbr0, #TTB_FLAGS_SMP)
+ ALT_UP(orr \ttbr0, \ttbr0, #TTB_FLAGS_UP)
+ ALT_SMP(orr \ttbr1, \ttbr1, #TTB_FLAGS_SMP)
+ ALT_UP(orr \ttbr1, \ttbr1, #TTB_FLAGS_UP)
+ mcr p15, 0, \ttbr1, c2, c0, 1 @ load TTB1
+ .endm
+
+ __CPUINIT
+
+ /* AT
+ * TFR EV X F I D LR S
+ * .EEE ..EE PUI. .T.T 4RVI ZWRS BLDP WCAM
+ * rxxx rrxx xxx0 0101 xxxx xxxx x111 xxxx < forced
+ * 1 0 110 0011 1100 .111 1101 < we want
+ */
+ .align 2
+ .type v7_crval, #object
+v7_crval:
+ crval clear=0x0120c302, mmuset=0x10c03c7d, ucset=0x00c01c7c
+
+ .previous
--- /dev/null
+/*
+ * arch/arm/mm/proc-v7-3level.S
+ *
+ * Copyright (C) 2001 Deep Blue Solutions Ltd.
+ * Copyright (C) 2011 ARM Ltd.
+ * Author: Catalin Marinas <catalin.marinas@arm.com>
+ * based on arch/arm/mm/proc-v7-2level.S
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#define TTB_IRGN_NC (0 << 8)
+#define TTB_IRGN_WBWA (1 << 8)
+#define TTB_IRGN_WT (2 << 8)
+#define TTB_IRGN_WB (3 << 8)
+#define TTB_RGN_NC (0 << 10)
+#define TTB_RGN_OC_WBWA (1 << 10)
+#define TTB_RGN_OC_WT (2 << 10)
+#define TTB_RGN_OC_WB (3 << 10)
+#define TTB_S (3 << 12)
+#define TTB_EAE (1 << 31)
+
+/* PTWs cacheable, inner WB not shareable, outer WB not shareable */
+#define TTB_FLAGS_UP (TTB_IRGN_WB|TTB_RGN_OC_WB)
+#define PMD_FLAGS_UP (PMD_SECT_WB)
+
+/* PTWs cacheable, inner WBWA shareable, outer WBWA not shareable */
+#define TTB_FLAGS_SMP (TTB_IRGN_WBWA|TTB_S|TTB_RGN_OC_WBWA)
+#define PMD_FLAGS_SMP (PMD_SECT_WBWA|PMD_SECT_S)
+
+/*
+ * cpu_v7_switch_mm(pgd_phys, tsk)
+ *
+ * Set the translation table base pointer to be pgd_phys (physical address of
+ * the new TTB).
+ */
+ENTRY(cpu_v7_switch_mm)
+#ifdef CONFIG_MMU
+ ldr r1, [r1, #MM_CONTEXT_ID] @ get mm->context.id
+ and r3, r1, #0xff
+ mov r3, r3, lsl #(48 - 32) @ ASID
+ mcrr p15, 0, r0, r3, c2 @ set TTB 0
+ isb
+#endif
+ mov pc, lr
+ENDPROC(cpu_v7_switch_mm)
+
+/*
+ * cpu_v7_set_pte_ext(ptep, pte)
+ *
+ * Set a level 2 translation table entry.
+ * - ptep - pointer to level 3 translation table entry
+ * - pte - PTE value to store (64-bit in r2 and r3)
+ */
+ENTRY(cpu_v7_set_pte_ext)
+#ifdef CONFIG_MMU
+ tst r2, #L_PTE_PRESENT
+ beq 1f
+ tst r3, #1 << (55 - 32) @ L_PTE_DIRTY
+ orreq r2, #L_PTE_RDONLY
+1: strd r2, r3, [r0]
+ mcr p15, 0, r0, c7, c10, 1 @ flush_pte
+#endif
+ mov pc, lr
+ENDPROC(cpu_v7_set_pte_ext)
+
+ /*
+ * Memory region attributes for LPAE (defined in pgtable-3level.h):
+ *
+ * n = AttrIndx[2:0]
+ *
+ * n MAIR
+ * UNCACHED 000 00000000
+ * BUFFERABLE 001 01000100
+ * DEV_WC 001 01000100
+ * WRITETHROUGH 010 10101010
+ * WRITEBACK 011 11101110
+ * DEV_CACHED 011 11101110
+ * DEV_SHARED 100 00000100
+ * DEV_NONSHARED 100 00000100
+ * unused 101
+ * unused 110
+ * WRITEALLOC 111 11111111
+ */
+.equ PRRR, 0xeeaa4400 @ MAIR0
+.equ NMRR, 0xff000004 @ MAIR1
+
+ /*
+ * Macro for setting up the TTBRx and TTBCR registers.
+ * - \ttbr1 updated.
+ */
+ .macro v7_ttb_setup, zero, ttbr0, ttbr1, tmp
+ ldr \tmp, =swapper_pg_dir @ swapper_pg_dir virtual address
+ cmp \ttbr1, \tmp @ PHYS_OFFSET > PAGE_OFFSET? (branch below)
+ mrc p15, 0, \tmp, c2, c0, 2 @ TTB control register
+ orr \tmp, \tmp, #TTB_EAE
+ ALT_SMP(orr \tmp, \tmp, #TTB_FLAGS_SMP)
+ ALT_UP(orr \tmp, \tmp, #TTB_FLAGS_UP)
+ ALT_SMP(orr \tmp, \tmp, #TTB_FLAGS_SMP << 16)
+ ALT_UP(orr \tmp, \tmp, #TTB_FLAGS_UP << 16)
+ /*
+ * TTBR0/TTBR1 split (PAGE_OFFSET):
+ * 0x40000000: T0SZ = 2, T1SZ = 0 (not used)
+ * 0x80000000: T0SZ = 0, T1SZ = 1
+ * 0xc0000000: T0SZ = 0, T1SZ = 2
+ *
+ * Only use this feature if PHYS_OFFSET <= PAGE_OFFSET, otherwise
+ * booting secondary CPUs would end up using TTBR1 for the identity
+ * mapping set up in TTBR0.
+ */
+ bhi 9001f @ PHYS_OFFSET > PAGE_OFFSET?
+ orr \tmp, \tmp, #(((PAGE_OFFSET >> 30) - 1) << 16) @ TTBCR.T1SZ
+#if defined CONFIG_VMSPLIT_2G
+ /* PAGE_OFFSET == 0x80000000, T1SZ == 1 */
+ add \ttbr1, \ttbr1, #1 << 4 @ skip two L1 entries
+#elif defined CONFIG_VMSPLIT_3G
+ /* PAGE_OFFSET == 0xc0000000, T1SZ == 2 */
+ add \ttbr1, \ttbr1, #4096 * (1 + 3) @ only L2 used, skip pgd+3*pmd
+#endif
+ /* CONFIG_VMSPLIT_1G does not need TTBR1 adjustment */
+9001: mcr p15, 0, \tmp, c2, c0, 2 @ TTB control register
+ mcrr p15, 1, \ttbr1, \zero, c2 @ load TTBR1
+ .endm
+
+ __CPUINIT
+
+ /*
+ * AT
+ * TFR EV X F IHD LR S
+ * .EEE ..EE PUI. .TAT 4RVI ZWRS BLDP WCAM
+ * rxxx rrxx xxx0 0101 xxxx xxxx x111 xxxx < forced
+ * 11 0 110 1 0011 1100 .111 1101 < we want
+ */
+ .align 2
+ .type v7_crval, #object
+v7_crval:
+ crval clear=0x0120c302, mmuset=0x30c23c7d, ucset=0x00c01c7c
+
+ .previous
#include "proc-macros.S"
-#define TTB_S (1 << 1)
-#define TTB_RGN_NC (0 << 3)
-#define TTB_RGN_OC_WBWA (1 << 3)
-#define TTB_RGN_OC_WT (2 << 3)
-#define TTB_RGN_OC_WB (3 << 3)
-#define TTB_NOS (1 << 5)
-#define TTB_IRGN_NC ((0 << 0) | (0 << 6))
-#define TTB_IRGN_WBWA ((0 << 0) | (1 << 6))
-#define TTB_IRGN_WT ((1 << 0) | (0 << 6))
-#define TTB_IRGN_WB ((1 << 0) | (1 << 6))
-
-/* PTWs cacheable, inner WB not shareable, outer WB not shareable */
-#define TTB_FLAGS_UP TTB_IRGN_WB|TTB_RGN_OC_WB
-#define PMD_FLAGS_UP PMD_SECT_WB
-
-/* PTWs cacheable, inner WBWA shareable, outer WBWA not shareable */
-#define TTB_FLAGS_SMP TTB_IRGN_WBWA|TTB_S|TTB_NOS|TTB_RGN_OC_WBWA
-#define PMD_FLAGS_SMP PMD_SECT_WBWA|PMD_SECT_S
+#ifdef CONFIG_ARM_LPAE
+#include "proc-v7-3level.S"
+#else
+#include "proc-v7-2level.S"
+#endif
ENTRY(cpu_v7_proc_init)
mov pc, lr
mov pc, lr
ENDPROC(cpu_v7_dcache_clean_area)
-/*
- * cpu_v7_switch_mm(pgd_phys, tsk)
- *
- * Set the translation table base pointer to be pgd_phys
- *
- * - pgd_phys - physical address of new TTB
- *
- * It is assumed that:
- * - we are not using split page tables
- */
-ENTRY(cpu_v7_switch_mm)
-#ifdef CONFIG_MMU
- mov r2, #0
- ldr r1, [r1, #MM_CONTEXT_ID] @ get mm->context.id
- ALT_SMP(orr r0, r0, #TTB_FLAGS_SMP)
- ALT_UP(orr r0, r0, #TTB_FLAGS_UP)
-#ifdef CONFIG_ARM_ERRATA_430973
- mcr p15, 0, r2, c7, c5, 6 @ flush BTAC/BTB
-#endif
-#ifdef CONFIG_ARM_ERRATA_754322
- dsb
-#endif
- mcr p15, 0, r2, c13, c0, 1 @ set reserved context ID
- isb
-1: mcr p15, 0, r0, c2, c0, 0 @ set TTB 0
- isb
-#ifdef CONFIG_ARM_ERRATA_754322
- dsb
-#endif
- mcr p15, 0, r1, c13, c0, 1 @ set context ID
- isb
-#endif
- mov pc, lr
-ENDPROC(cpu_v7_switch_mm)
-
-/*
- * cpu_v7_set_pte_ext(ptep, pte)
- *
- * Set a level 2 translation table entry.
- *
- * - ptep - pointer to level 2 translation table entry
- * (hardware version is stored at +2048 bytes)
- * - pte - PTE value to store
- * - ext - value for extended PTE bits
- */
-ENTRY(cpu_v7_set_pte_ext)
-#ifdef CONFIG_MMU
- str r1, [r0] @ linux version
-
- bic r3, r1, #0x000003f0
- bic r3, r3, #PTE_TYPE_MASK
- orr r3, r3, r2
- orr r3, r3, #PTE_EXT_AP0 | 2
-
- tst r1, #1 << 4
- orrne r3, r3, #PTE_EXT_TEX(1)
-
- eor r1, r1, #L_PTE_DIRTY
- tst r1, #L_PTE_RDONLY | L_PTE_DIRTY
- orrne r3, r3, #PTE_EXT_APX
-
- tst r1, #L_PTE_USER
- orrne r3, r3, #PTE_EXT_AP1
-#ifdef CONFIG_CPU_USE_DOMAINS
- @ allow kernel read/write access to read-only user pages
- tstne r3, #PTE_EXT_APX
- bicne r3, r3, #PTE_EXT_APX | PTE_EXT_AP0
-#endif
-
- tst r1, #L_PTE_XN
- orrne r3, r3, #PTE_EXT_XN
-
- tst r1, #L_PTE_YOUNG
- tstne r1, #L_PTE_PRESENT
- moveq r3, #0
-
- ARM( str r3, [r0, #2048]! )
- THUMB( add r0, r0, #2048 )
- THUMB( str r3, [r0] )
- mcr p15, 0, r0, c7, c10, 1 @ flush_pte
-#endif
- mov pc, lr
-ENDPROC(cpu_v7_set_pte_ext)
-
string cpu_v7_name, "ARMv7 Processor"
.align
- /*
- * Memory region attributes with SCTLR.TRE=1
- *
- * n = TEX[0],C,B
- * TR = PRRR[2n+1:2n] - memory type
- * IR = NMRR[2n+1:2n] - inner cacheable property
- * OR = NMRR[2n+17:2n+16] - outer cacheable property
- *
- * n TR IR OR
- * UNCACHED 000 00
- * BUFFERABLE 001 10 00 00
- * WRITETHROUGH 010 10 10 10
- * WRITEBACK 011 10 11 11
- * reserved 110
- * WRITEALLOC 111 10 01 01
- * DEV_SHARED 100 01
- * DEV_NONSHARED 100 01
- * DEV_WC 001 10
- * DEV_CACHED 011 10
- *
- * Other attributes:
- *
- * DS0 = PRRR[16] = 0 - device shareable property
- * DS1 = PRRR[17] = 1 - device shareable property
- * NS0 = PRRR[18] = 0 - normal shareable property
- * NS1 = PRRR[19] = 1 - normal shareable property
- * NOS = PRRR[24+n] = 1 - not outer shareable
- */
-.equ PRRR, 0xff0a81a8
-.equ NMRR, 0x40e040e0
-
/* Suspend/resume support: derived from arch/arm/mach-s5pv210/sleep.S */
.globl cpu_v7_suspend_size
-.equ cpu_v7_suspend_size, 4 * 7
+.equ cpu_v7_suspend_size, 4 * 8
#ifdef CONFIG_ARM_CPU_SUSPEND
ENTRY(cpu_v7_do_suspend)
stmfd sp!, {r4 - r10, lr}
stmia r0!, {r4 - r5}
mrc p15, 0, r6, c3, c0, 0 @ Domain ID
mrc p15, 0, r7, c2, c0, 1 @ TTB 1
+ mrc p15, 0, r11, c2, c0, 2 @ TTB control register
mrc p15, 0, r8, c1, c0, 0 @ Control register
mrc p15, 0, r9, c1, c0, 1 @ Auxiliary control register
mrc p15, 0, r10, c1, c0, 2 @ Co-processor access control
- stmia r0, {r6 - r10}
+ stmia r0, {r6 - r11}
ldmfd sp!, {r4 - r10, pc}
ENDPROC(cpu_v7_do_suspend)
ldmia r0!, {r4 - r5}
mcr p15, 0, r4, c13, c0, 0 @ FCSE/PID
mcr p15, 0, r5, c13, c0, 3 @ User r/o thread ID
- ldmia r0, {r6 - r10}
+ ldmia r0, {r6 - r11}
mcr p15, 0, r6, c3, c0, 0 @ Domain ID
+#ifndef CONFIG_ARM_LPAE
ALT_SMP(orr r1, r1, #TTB_FLAGS_SMP)
ALT_UP(orr r1, r1, #TTB_FLAGS_UP)
+#endif
mcr p15, 0, r1, c2, c0, 0 @ TTB 0
mcr p15, 0, r7, c2, c0, 1 @ TTB 1
- mcr p15, 0, ip, c2, c0, 2 @ TTB control register
+ mcr p15, 0, r11, c2, c0, 2 @ TTB control register
mrc p15, 0, r4, c1, c0, 1 @ Read Auxiliary control register
teq r4, r9 @ Is it already set?
mcrne p15, 0, r9, c1, c0, 1 @ No, so write it
dsb
#ifdef CONFIG_MMU
mcr p15, 0, r10, c8, c7, 0 @ invalidate I + D TLBs
- mcr p15, 0, r10, c2, c0, 2 @ TTB control register
- ALT_SMP(orr r4, r4, #TTB_FLAGS_SMP)
- ALT_UP(orr r4, r4, #TTB_FLAGS_UP)
- ALT_SMP(orr r8, r8, #TTB_FLAGS_SMP)
- ALT_UP(orr r8, r8, #TTB_FLAGS_UP)
- mcr p15, 0, r8, c2, c0, 1 @ load TTB1
+ v7_ttb_setup r10, r4, r8, r5 @ TTBCR, TTBRx setup
ldr r5, =PRRR @ PRRR
ldr r6, =NMRR @ NMRR
mcr p15, 0, r5, c10, c2, 0 @ write PRRR
mov pc, lr @ return to head.S:__ret
ENDPROC(__v7_setup)
- /* AT
- * TFR EV X F I D LR S
- * .EEE ..EE PUI. .T.T 4RVI ZWRS BLDP WCAM
- * rxxx rrxx xxx0 0101 xxxx xxxx x111 xxxx < forced
- * 1 0 110 0011 1100 .111 1101 < we want
- */
- .type v7_crval, #object
-v7_crval:
- crval clear=0x0120c302, mmuset=0x10c03c7d, ucset=0x00c01c7c
-
+ .align 2
__v7_setup_stack:
.space 4 * 11 @ 11 registers
*/
.macro __v7_proc initfunc, mm_mmuflags = 0, io_mmuflags = 0, hwcaps = 0
ALT_SMP(.long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | \
- PMD_FLAGS_SMP | \mm_mmuflags)
+ PMD_SECT_AF | PMD_FLAGS_SMP | \mm_mmuflags)
ALT_UP(.long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | \
- PMD_FLAGS_UP | \mm_mmuflags)
- .long PMD_TYPE_SECT | PMD_SECT_XN | PMD_SECT_AP_WRITE | \
- PMD_SECT_AP_READ | \io_mmuflags
+ PMD_SECT_AF | PMD_FLAGS_UP | \mm_mmuflags)
+ .long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | \
+ PMD_SECT_AP_READ | PMD_SECT_AF | \io_mmuflags
W(b) \initfunc
.long cpu_arch_name
.long cpu_elf_name
.long v7_cache_fns
.endm
+#ifndef CONFIG_ARM_LPAE
/*
* ARM Ltd. Cortex A5 processor.
*/
.long 0xff0ffff0
__v7_proc __v7_ca9mp_setup
.size __v7_ca9mp_proc_info, . - __v7_ca9mp_proc_info
+#endif /* CONFIG_ARM_LPAE */
/*
* ARM Ltd. Cortex A15 processor.
projects / karo-tx-linux.git / commitdiff