X-Git-Url: https://git.kernelconcepts.de/?a=blobdiff_plain;f=arch%2Farm%2Fmach-imx%2Fbusfreq_ddr3.c;h=b84c9d82049137557a1ca51c3634c74d1f1040af;hb=312b639044a982a438545b04acfebede7cb45e4a;hp=6ecd2f8a4407fe0e1fc650761c647b75dd003542;hpb=4ba3f76ddf029783df700fa559289f5eedd5903d;p=karo-tx-linux.git diff --git a/arch/arm/mach-imx/busfreq_ddr3.c b/arch/arm/mach-imx/busfreq_ddr3.c index 6ecd2f8a4407..b84c9d820491 100644 --- a/arch/arm/mach-imx/busfreq_ddr3.c +++ b/arch/arm/mach-imx/busfreq_ddr3.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011-2013 Freescale Semiconductor, Inc. All Rights Reserved. + * Copyright (C) 2011-2014 Freescale Semiconductor, Inc. All Rights Reserved. */ /* @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -42,6 +43,26 @@ #include "hardware.h" + +/* + * This structure is for passing necessary data for low level ocram + * busfreq code(arch/arm/mach-imx/ddr3_freq_imx6.S), if this struct + * definition is changed, the offset definition in + * arch/arm/mach-imx/ddr3_freq_imx6.S must be also changed accordingly, + * otherwise, the busfreq change function will be broken! + * + * This structure will be placed in front of the asm code on ocram. + */ +struct imx6_busfreq_info { + u32 freq; + void *ddr_settings; + u32 dll_off; + void *iomux_offsets; + u32 mu_delay_val; +} __aligned(8); + +static struct imx6_busfreq_info *imx6sx_busfreq_info; + /* DDR settings */ static unsigned long (*iram_ddr_settings)[2]; static unsigned long (*normal_mmdc_settings)[2]; @@ -49,10 +70,14 @@ static unsigned long (*iram_iomux_settings)[2]; static void __iomem *mmdc_base; static void __iomem *iomux_base; -static void __iomem *ccm_base; -static void __iomem *l2_base; static void __iomem *gic_dist_base; -static u32 *irqs_used; + +static int ddr_settings_size; +static int iomux_settings_size; +static int curr_ddr_rate; + +void (*imx6sx_change_ddr_freq)(struct imx6_busfreq_info *busfreq_info); +extern void imx6sx_ddr3_freq_change(struct imx6_busfreq_info *busfreq_info); void (*mx6_change_ddr_freq)(u32 freq, void *ddr_settings, bool dll_mode, void *iomux_offsets) = NULL; @@ -64,16 +89,59 @@ extern int audio_bus_freq_mode; extern void mx6_ddr3_freq_change(u32 freq, void *ddr_settings, bool dll_mode, void *iomux_offsets); -static void *ddr_freq_change_iram_base; -static int ddr_settings_size; -static int iomux_settings_size; -static volatile unsigned int cpus_in_wfe; -static volatile bool wait_for_ddr_freq_update; -static int curr_ddr_rate; +extern unsigned long save_ttbr1(void); +extern void restore_ttbr1(unsigned long ttbr1); +extern unsigned long ddr_freq_change_iram_base; + +extern unsigned long ddr_freq_change_total_size; +extern unsigned long iram_tlb_phys_addr; + +extern unsigned long mx6_ddr3_freq_change_start asm("mx6_ddr3_freq_change_start"); +extern unsigned long mx6_ddr3_freq_change_end asm("mx6_ddr3_freq_change_end"); +extern unsigned long imx6sx_ddr3_freq_change_start asm("imx6sx_ddr3_freq_change_start"); +extern unsigned long imx6sx_ddr3_freq_change_end asm("imx6sx_ddr3_freq_change_end"); +#ifdef CONFIG_SMP +static unsigned long wfe_freq_change_iram_base; +u32 *wait_for_ddr_freq_update; +static unsigned int online_cpus; +static u32 *irqs_used; + +void (*wfe_change_ddr_freq)(u32 cpuid, u32 *ddr_freq_change_done); +extern void wfe_ddr3_freq_change(u32 cpuid, u32 *ddr_freq_change_done); +extern unsigned long wfe_ddr3_freq_change_start asm("wfe_ddr3_freq_change_start"); +extern unsigned long wfe_ddr3_freq_change_end asm("wfe_ddr3_freq_change_end"); +extern void __iomem *imx_scu_base; +#endif #define MIN_DLL_ON_FREQ 333000000 #define MAX_DLL_OFF_FREQ 125000000 -#define DDR_FREQ_CHANGE_SIZE 0x2000 +#define MMDC0_MPMUR0 0x8b8 +#define MMDC0_MPMUR0_OFFSET 16 +#define MMDC0_MPMUR0_MASK 0x3ff + +unsigned long ddr3_dll_mx6sx[][2] = { + {0x0c, 0x0}, + {0x10, 0x0}, + {0x1C, 0x04008032}, + {0x1C, 0x00048031}, + {0x1C, 0x05208030}, + {0x1C, 0x04008040}, + {0x818, 0x0}, +}; + +unsigned long ddr3_calibration_mx6sx[][2] = { + {0x83c, 0x0}, + {0x840, 0x0}, + {0x848, 0x0}, + {0x850, 0x0}, +}; + +unsigned long iomux_offsets_mx6sx[][2] = { + {0x330, 0x0}, + {0x334, 0x0}, + {0x338, 0x0}, + {0x33c, 0x0}, +}; unsigned long ddr3_dll_mx6q[][2] = { {0x0c, 0x0}, @@ -144,6 +212,7 @@ int can_change_ddr_freq(void) return 1; } +#ifdef CONFIG_SMP /* * each active core apart from the one changing * the DDR frequency will execute this function. @@ -152,27 +221,79 @@ int can_change_ddr_freq(void) */ irqreturn_t wait_in_wfe_irq(int irq, void *dev_id) { - u32 me = smp_processor_id(); - - *((char *)(&cpus_in_wfe) + (u8)me) = 0xff; + u32 me; - while (wait_for_ddr_freq_update) - wfe(); + me = smp_processor_id(); +#ifdef CONFIG_LOCAL_TIMERS + clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, + &me); +#endif + wfe_change_ddr_freq(0xff << (me * 8), (u32 *)&iram_iomux_settings[0][1]); - *((char *)(&cpus_in_wfe) + (u8)me) = 0; +#ifdef CONFIG_LOCAL_TIMERS + clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, + &me); +#endif return IRQ_HANDLED; } +#endif + +int update_ddr_freq_imx6sx(int ddr_rate) +{ + int i; + bool dll_off = false; + unsigned long ttbr1; + + if (ddr_rate == curr_ddr_rate) + return 0; + + printk(KERN_DEBUG "\nBus freq set to %d start...\n", ddr_rate); + + if (low_bus_freq_mode || audio_bus_freq_mode) + dll_off = true; + + imx6sx_busfreq_info->dll_off = dll_off; + iram_ddr_settings[0][0] = ddr_settings_size; + iram_iomux_settings[0][0] = iomux_settings_size; + for (i = 0; i < iram_ddr_settings[0][0]; i++) { + iram_ddr_settings[i + 1][0] = + normal_mmdc_settings[i][0]; + iram_ddr_settings[i + 1][1] = + normal_mmdc_settings[i][1]; + } + + local_irq_disable(); + + ttbr1 = save_ttbr1(); + imx6sx_busfreq_info->freq = ddr_rate; + imx6sx_busfreq_info->ddr_settings = iram_ddr_settings; + imx6sx_busfreq_info->iomux_offsets = iram_iomux_settings; + imx6sx_busfreq_info->mu_delay_val = ((readl_relaxed(mmdc_base + MMDC0_MPMUR0) + >> MMDC0_MPMUR0_OFFSET) & MMDC0_MPMUR0_MASK); + + imx6sx_change_ddr_freq(imx6sx_busfreq_info); + restore_ttbr1(ttbr1); + curr_ddr_rate = ddr_rate; + + local_irq_enable(); + + printk(KERN_DEBUG "Bus freq set to %d done!\n", ddr_rate); + + return 0; +} /* change the DDR frequency. */ -int update_ddr_freq(int ddr_rate) +int update_ddr_freq_imx6q(int ddr_rate) { int i, j; - unsigned int reg; bool dll_off = false; - unsigned int online_cpus = 0; + int me = 0; + unsigned long ttbr1; +#ifdef CONFIG_SMP + unsigned int reg; int cpu = 0; - int me; +#endif if (!can_change_ddr_freq()) return -1; @@ -213,12 +334,28 @@ int update_ddr_freq(int ddr_rate) /* ensure that all Cores are in WFE. */ local_irq_disable(); +#ifdef CONFIG_SMP me = smp_processor_id(); - *((char *)(&cpus_in_wfe) + (u8)me) = 0xff; - wait_for_ddr_freq_update = true; + /* Make sure all the online cores are active */ + while (1) { + bool not_exited_busfreq = false; + for_each_online_cpu(cpu) { + u32 reg = __raw_readl(imx_scu_base + 0x08); + if (reg & (0x02 << (cpu * 8))) + not_exited_busfreq = true; + } + if (!not_exited_busfreq) + break; + } + + wmb(); + *wait_for_ddr_freq_update = 1; + dsb(); + + online_cpus = readl_relaxed(imx_scu_base + 0x08); for_each_online_cpu(cpu) { - *((char *)(&online_cpus) + (u8)cpu) = 0xff; + *((char *)(&online_cpus) + (u8)cpu) = 0x02; if (cpu != me) { /* set the interrupt to be pending in the GIC. */ reg = 1 << (irqs_used[cpu] % 32); @@ -226,77 +363,157 @@ int update_ddr_freq(int ddr_rate) + (irqs_used[cpu] / 32) * 4); } } - while (cpus_in_wfe != online_cpus) - udelay(5); + /* Wait for the other active CPUs to idle */ + while (1) { + u32 reg = readl_relaxed(imx_scu_base + 0x08); + reg |= (0x02 << (me * 8)); + if (reg == online_cpus) + break; + } +#endif + + /* Ensure iram_tlb_phys_addr is flushed to DDR. */ + __cpuc_flush_dcache_area(&iram_tlb_phys_addr, sizeof(iram_tlb_phys_addr)); + outer_clean_range(virt_to_phys(&iram_tlb_phys_addr), virt_to_phys(&iram_tlb_phys_addr + 1)); + ttbr1 = save_ttbr1(); /* Now we can change the DDR frequency. */ mx6_change_ddr_freq(ddr_rate, iram_ddr_settings, dll_off, iram_iomux_settings); - + restore_ttbr1(ttbr1); curr_ddr_rate = ddr_rate; +#ifdef CONFIG_SMP + wmb(); /* DDR frequency change is done . */ - wait_for_ddr_freq_update = false; + *wait_for_ddr_freq_update = 0; + dsb(); /* wake up all the cores. */ sev(); - - *((char *)(&cpus_in_wfe) + (u8)me) = 0; +#endif local_irq_enable(); - printk(KERN_DEBUG "Bus freq set to %d done!\n", ddr_rate); + printk(KERN_DEBUG "Bus freq set to %d done! cpu=%d\n", ddr_rate, me); return 0; } -int init_mmdc_settings(struct platform_device *busfreq_pdev) +int init_mmdc_ddr3_settings_imx6sx(struct platform_device *busfreq_pdev) { - struct device *dev = &busfreq_pdev->dev; - struct platform_device *ocram_dev; - unsigned int iram_paddr; - int i, err; - u32 cpu; + int i; struct device_node *node; - struct gen_pool *iram_pool; - void *iram_addr; + unsigned long ddr_code_size; - node = of_find_compatible_node(NULL, NULL, "fsl,imx6q-mmdc-combine"); + node = of_find_compatible_node(NULL, NULL, "fsl,imx6q-mmdc"); if (!node) { - printk(KERN_ERR "failed to find imx6q-mmdc device tree data!\n"); + printk(KERN_ERR "failed to find mmdc device tree data!\n"); return -EINVAL; } mmdc_base = of_iomap(node, 0); WARN(!mmdc_base, "unable to map mmdc registers\n"); - node = NULL; - if (cpu_is_imx6q()) - node = of_find_compatible_node(NULL, NULL, "fsl,imx6q-iomuxc"); - if (cpu_is_imx6dl()) - node = of_find_compatible_node(NULL, NULL, - "fsl,imx6dl-iomuxc"); + node = of_find_compatible_node(NULL, NULL, "fsl,imx6sx-iomuxc"); if (!node) { - printk(KERN_ERR "failed to find imx6q-iomux device tree data!\n"); + printk(KERN_ERR "failed to find iomuxc device tree data!\n"); return -EINVAL; } iomux_base = of_iomap(node, 0); WARN(!iomux_base, "unable to map iomux registers\n"); - node = of_find_compatible_node(NULL, NULL, "fsl,imx6q-ccm"); + ddr_settings_size = ARRAY_SIZE(ddr3_dll_mx6sx) + + ARRAY_SIZE(ddr3_calibration_mx6sx); + + normal_mmdc_settings = kmalloc((ddr_settings_size * 8), GFP_KERNEL); + memcpy(normal_mmdc_settings, ddr3_dll_mx6sx, + sizeof(ddr3_dll_mx6sx)); + memcpy(((char *)normal_mmdc_settings + sizeof(ddr3_dll_mx6sx)), + ddr3_calibration_mx6sx, sizeof(ddr3_calibration_mx6sx)); + + /* store the original DDR settings at boot. */ + for (i = 0; i < ddr_settings_size; i++) { + /* + * writes via command mode register cannot be read back. + * hence hardcode them in the initial static array. + * this may require modification on a per customer basis. + */ + if (normal_mmdc_settings[i][0] != 0x1C) + normal_mmdc_settings[i][1] = + readl_relaxed(mmdc_base + + normal_mmdc_settings[i][0]); + } + + iomux_settings_size = ARRAY_SIZE(iomux_offsets_mx6sx); + + ddr_code_size = (&imx6sx_ddr3_freq_change_end -&imx6sx_ddr3_freq_change_start) *4 + + sizeof(*imx6sx_busfreq_info); + + imx6sx_busfreq_info = (struct imx6_busfreq_info *)ddr_freq_change_iram_base; + + imx6sx_change_ddr_freq = (void *)fncpy((void *)ddr_freq_change_iram_base + sizeof(*imx6sx_busfreq_info), + &imx6sx_ddr3_freq_change, ddr_code_size - sizeof(*imx6sx_busfreq_info)); + + /* + * Store the size of the array in iRAM also, + * increase the size by 8 bytes. + */ + iram_iomux_settings = (void *)(ddr_freq_change_iram_base + ddr_code_size); + iram_ddr_settings = iram_iomux_settings + (iomux_settings_size * 8) + 8; + + if ((ddr_code_size + (iomux_settings_size + ddr_settings_size) * 8 + 16) + > ddr_freq_change_total_size) { + printk(KERN_ERR "Not enough memory allocated for DDR Frequency change code.\n"); + return EINVAL; + } + + for (i = 0; i < iomux_settings_size; i++) { + iomux_offsets_mx6sx[i][1] = + readl_relaxed(iomux_base + + iomux_offsets_mx6sx[i][0]); + iram_iomux_settings[i + 1][0] = + iomux_offsets_mx6sx[i][0]; + iram_iomux_settings[i + 1][1] = + iomux_offsets_mx6sx[i][1]; + } + + curr_ddr_rate = ddr_normal_rate; + + return 0; +} + +int init_mmdc_ddr3_settings_imx6q(struct platform_device *busfreq_pdev) +{ + int i; + struct device_node *node; + unsigned long ddr_code_size; + unsigned long wfe_code_size = 0; +#ifdef CONFIG_SMP + u32 cpu; + struct device *dev = &busfreq_pdev->dev; + int err; +#endif + + node = of_find_compatible_node(NULL, NULL, "fsl,imx6q-mmdc-combine"); if (!node) { - printk(KERN_ERR "failed to find imx6q-ccm device tree data!\n"); + printk(KERN_ERR "failed to find imx6q-mmdc device tree data!\n"); return -EINVAL; } - ccm_base = of_iomap(node, 0); + mmdc_base = of_iomap(node, 0); WARN(!mmdc_base, "unable to map mmdc registers\n"); - node = of_find_compatible_node(NULL, NULL, "arm,pl310-cache"); + node = NULL; + if (cpu_is_imx6q()) + node = of_find_compatible_node(NULL, NULL, "fsl,imx6q-iomuxc"); + if (cpu_is_imx6dl()) + node = of_find_compatible_node(NULL, NULL, + "fsl,imx6dl-iomuxc"); if (!node) { - printk(KERN_ERR "failed to find imx6q-pl310-cache device tree data!\n"); + printk(KERN_ERR "failed to find imx6q-iomux device tree data!\n"); return -EINVAL; } - l2_base = of_iomap(node, 0); - WARN(!mmdc_base, "unable to map mmdc registers\n"); + iomux_base = of_iomap(node, 0); + WARN(!iomux_base, "unable to map iomux registers\n"); node = NULL; node = of_find_compatible_node(NULL, NULL, "arm,cortex-a9-gic"); @@ -340,10 +557,11 @@ int init_mmdc_settings(struct platform_device *busfreq_pdev) + normal_mmdc_settings[i][0]); } +#ifdef CONFIG_SMP irqs_used = devm_kzalloc(dev, sizeof(u32) * num_present_cpus(), GFP_KERNEL); - for_each_present_cpu(cpu) { + for_each_online_cpu(cpu) { int irq; /* @@ -369,61 +587,41 @@ int init_mmdc_settings(struct platform_device *busfreq_pdev) } irqs_used[cpu] = irq; } +#endif + iomux_settings_size = ARRAY_SIZE(iomux_offsets_mx6q); - node = NULL; - node = of_find_compatible_node(NULL, NULL, "mmio-sram"); - if (!node) { - dev_err(dev, "%s: failed to find ocram node\n", - __func__); - return -EINVAL; - } + ddr_code_size = (&mx6_ddr3_freq_change_end -&mx6_ddr3_freq_change_start) *4; - ocram_dev = of_find_device_by_node(node); - if (!ocram_dev) { - dev_err(dev, "failed to find ocram device!\n"); - return -EINVAL; - } + mx6_change_ddr_freq = (void *)fncpy((void *)ddr_freq_change_iram_base, + &mx6_ddr3_freq_change, ddr_code_size); - iram_pool = dev_get_gen_pool(&ocram_dev->dev); - if (!iram_pool) { - dev_err(dev, "iram pool unavailable!\n"); - return -EINVAL; - } + /* + * Store the size of the array in iRAM also, + * increase the size by 8 bytes. + */ + iram_iomux_settings = (void *)(ddr_freq_change_iram_base + ddr_code_size); + iram_ddr_settings = iram_iomux_settings + (iomux_settings_size * 8) + 8; - iomux_settings_size = ARRAY_SIZE(iomux_offsets_mx6q); - iram_addr = (void *)gen_pool_alloc(iram_pool, - (iomux_settings_size * 8) + 8); - iram_iomux_settings = iram_addr; - if (!iram_iomux_settings) { - dev_err(dev, "unable to alloc iram for IOMUX settings!\n"); - return -ENOMEM; - } +#ifdef CONFIG_SMP + wfe_freq_change_iram_base = (unsigned long)((u32 *)iram_ddr_settings + (ddr_settings_size * 8) + 8); - /* - * Allocate extra space to store the number of entries in the - * ddr_settings plus 4 extra regsiter information that needs - * to be passed to the frequency change code. - * sizeof(iram_ddr_settings) = sizeof(ddr_settings) + - * entries in ddr_settings + 16. - * The last 4 enties store the addresses of the registers: - * CCM_BASE_ADDR - * MMDC_BASE_ADDR - * IOMUX_BASE_ADDR - * L2X0_BASE_ADDR - */ - iram_addr = (void *)gen_pool_alloc(iram_pool, - (ddr_settings_size * 8) + 8 + 32); - iram_ddr_settings = iram_addr; - if (!iram_ddr_settings) { - dev_err(dev, "unable to alloc iram for ddr settings!\n"); - return -ENOMEM; - } + if (wfe_freq_change_iram_base & (FNCPY_ALIGN - 1)) + wfe_freq_change_iram_base += FNCPY_ALIGN - ((uintptr_t)wfe_freq_change_iram_base % (FNCPY_ALIGN)); - i = ddr_settings_size + 1; - iram_ddr_settings[i][0] = (unsigned long)mmdc_base; - iram_ddr_settings[i+1][0] = (unsigned long)ccm_base; - iram_ddr_settings[i+2][0] = (unsigned long)iomux_base; - iram_ddr_settings[i+3][0] = (unsigned long)l2_base; + wfe_code_size = (&wfe_ddr3_freq_change_end -&wfe_ddr3_freq_change_start) *4; + + wfe_change_ddr_freq = (void *)fncpy((void *)wfe_freq_change_iram_base, + &wfe_ddr3_freq_change, wfe_code_size); + + /* Store the variable used to communicate between cores in a non-cacheable IRAM area */ + wait_for_ddr_freq_update = (u32 *)&iram_iomux_settings[0][1]; +#endif + + if ((ddr_code_size + wfe_code_size + (iomux_settings_size + ddr_settings_size) * 8 + 16) + > ddr_freq_change_total_size) { + printk(KERN_ERR "Not enough memory allocated for DDR Frequency change code.\n"); + return EINVAL; + } if (cpu_is_imx6q()) { /* store the IOMUX settings at boot. */ @@ -446,25 +644,6 @@ int init_mmdc_settings(struct platform_device *busfreq_pdev) } } - ddr_freq_change_iram_base = (void *)gen_pool_alloc(iram_pool, - DDR_FREQ_CHANGE_SIZE); - if (!ddr_freq_change_iram_base) { - dev_err(dev, "Cannot alloc iram for ddr freq change code!\n"); - return -ENOMEM; - } - - iram_paddr = gen_pool_virt_to_phys(iram_pool, - (unsigned long)ddr_freq_change_iram_base); - /* - * need to remap the area here since we want - * the memory region to be executable. - */ - ddr_freq_change_iram_base = __arm_ioremap(iram_paddr, - DDR_FREQ_CHANGE_SIZE, - MT_MEMORY_NONCACHED); - mx6_change_ddr_freq = (void *)fncpy(ddr_freq_change_iram_base, - &mx6_ddr3_freq_change, DDR_FREQ_CHANGE_SIZE); - curr_ddr_rate = ddr_normal_rate; return 0;