1 /* Kernel module help for PPC64.
2 Copyright (C) 2001, 2003 Rusty Russell IBM Corporation.
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 #include <linux/module.h>
19 #include <linux/elf.h>
20 #include <linux/moduleloader.h>
21 #include <linux/err.h>
22 #include <linux/vmalloc.h>
23 #include <linux/ftrace.h>
24 #include <linux/bug.h>
25 #include <asm/module.h>
26 #include <asm/firmware.h>
27 #include <asm/code-patching.h>
28 #include <linux/sort.h>
32 /* FIXME: We don't do .init separately. To do this, we'd need to have
33 a separate r2 value in the init and core section, and stub between
36 Using a magic allocator which places modules within 32MB solves
37 this, and makes other things simpler. Anton?
42 #define DEBUGP(fmt , ...)
45 /* Like PPC32, we need little trampolines to do > 24-bit jumps (into
46 the kernel itself). But on PPC64, these need to be used for every
47 jump, actually, to reset r2 (TOC+0x8000). */
48 struct ppc64_stub_entry
50 /* 28 byte jump instruction sequence (7 instructions) */
51 unsigned char jump[28];
52 unsigned char unused[4];
53 /* Data for the above code */
54 struct ppc64_opd_entry opd;
57 /* We use a stub to fix up r2 (TOC ptr) and to jump to the (external)
58 function which may be more than 24-bits away. We could simply
59 patch the new r2 value and function pointer into the stub, but it's
60 significantly shorter to put these values at the end of the stub
61 code, and patch the stub address (32-bits relative to the TOC ptr,
63 static struct ppc64_stub_entry ppc64_stub =
65 #ifdef __LITTLE_ENDIAN__
66 0x00, 0x00, 0x82, 0x3d, /* addis r12,r2, <high> */
67 0x00, 0x00, 0x8c, 0x39, /* addi r12,r12, <low> */
68 /* Save current r2 value in magic place on the stack. */
69 0x28, 0x00, 0x41, 0xf8, /* std r2,40(r1) */
70 0x20, 0x00, 0x6c, 0xe9, /* ld r11,32(r12) */
71 0x28, 0x00, 0x4c, 0xe8, /* ld r2,40(r12) */
72 0xa6, 0x03, 0x69, 0x7d, /* mtctr r11 */
73 0x20, 0x04, 0x80, 0x4e /* bctr */
75 0x3d, 0x82, 0x00, 0x00, /* addis r12,r2, <high> */
76 0x39, 0x8c, 0x00, 0x00, /* addi r12,r12, <low> */
77 /* Save current r2 value in magic place on the stack. */
78 0xf8, 0x41, 0x00, 0x28, /* std r2,40(r1) */
79 0xe9, 0x6c, 0x00, 0x20, /* ld r11,32(r12) */
80 0xe8, 0x4c, 0x00, 0x28, /* ld r2,40(r12) */
81 0x7d, 0x69, 0x03, 0xa6, /* mtctr r11 */
82 0x4e, 0x80, 0x04, 0x20 /* bctr */
86 /* Count how many different 24-bit relocations (different symbol,
88 static unsigned int count_relocs(const Elf64_Rela *rela, unsigned int num)
90 unsigned int i, r_info, r_addend, _count_relocs;
92 /* FIXME: Only count external ones --RR */
96 for (i = 0; i < num; i++)
97 /* Only count 24-bit relocs, others don't need stubs */
98 if (ELF64_R_TYPE(rela[i].r_info) == R_PPC_REL24 &&
99 (r_info != ELF64_R_SYM(rela[i].r_info) ||
100 r_addend != rela[i].r_addend)) {
102 r_info = ELF64_R_SYM(rela[i].r_info);
103 r_addend = rela[i].r_addend;
106 return _count_relocs;
109 static int relacmp(const void *_x, const void *_y)
111 const Elf64_Rela *x, *y;
113 y = (Elf64_Rela *)_x;
114 x = (Elf64_Rela *)_y;
116 /* Compare the entire r_info (as opposed to ELF64_R_SYM(r_info) only) to
117 * make the comparison cheaper/faster. It won't affect the sorting or
118 * the counting algorithms' performance
120 if (x->r_info < y->r_info)
122 else if (x->r_info > y->r_info)
124 else if (x->r_addend < y->r_addend)
126 else if (x->r_addend > y->r_addend)
132 static void relaswap(void *_x, void *_y, int size)
134 uint64_t *x, *y, tmp;
140 for (i = 0; i < sizeof(Elf64_Rela) / sizeof(uint64_t); i++) {
147 /* Get size of potential trampolines required. */
148 static unsigned long get_stubs_size(const Elf64_Ehdr *hdr,
149 const Elf64_Shdr *sechdrs)
151 /* One extra reloc so it's always 0-funcaddr terminated */
152 unsigned long relocs = 1;
155 /* Every relocated section... */
156 for (i = 1; i < hdr->e_shnum; i++) {
157 if (sechdrs[i].sh_type == SHT_RELA) {
158 DEBUGP("Found relocations in section %u\n", i);
159 DEBUGP("Ptr: %p. Number: %lu\n",
160 (void *)sechdrs[i].sh_addr,
161 sechdrs[i].sh_size / sizeof(Elf64_Rela));
163 /* Sort the relocation information based on a symbol and
164 * addend key. This is a stable O(n*log n) complexity
165 * alogrithm but it will reduce the complexity of
166 * count_relocs() to linear complexity O(n)
168 sort((void *)sechdrs[i].sh_addr,
169 sechdrs[i].sh_size / sizeof(Elf64_Rela),
170 sizeof(Elf64_Rela), relacmp, relaswap);
172 relocs += count_relocs((void *)sechdrs[i].sh_addr,
174 / sizeof(Elf64_Rela));
178 #ifdef CONFIG_DYNAMIC_FTRACE
179 /* make the trampoline to the ftrace_caller */
183 DEBUGP("Looks like a total of %lu stubs, max\n", relocs);
184 return relocs * sizeof(struct ppc64_stub_entry);
187 static void dedotify_versions(struct modversion_info *vers,
190 struct modversion_info *end;
192 for (end = (void *)vers + size; vers < end; vers++)
193 if (vers->name[0] == '.')
194 memmove(vers->name, vers->name+1, strlen(vers->name));
197 /* Undefined symbols which refer to .funcname, hack to funcname */
198 static void dedotify(Elf64_Sym *syms, unsigned int numsyms, char *strtab)
202 for (i = 1; i < numsyms; i++) {
203 if (syms[i].st_shndx == SHN_UNDEF) {
204 char *name = strtab + syms[i].st_name;
206 memmove(name, name+1, strlen(name));
211 int module_frob_arch_sections(Elf64_Ehdr *hdr,
218 /* Find .toc and .stubs sections, symtab and strtab */
219 for (i = 1; i < hdr->e_shnum; i++) {
221 if (strcmp(secstrings + sechdrs[i].sh_name, ".stubs") == 0)
222 me->arch.stubs_section = i;
223 else if (strcmp(secstrings + sechdrs[i].sh_name, ".toc") == 0)
224 me->arch.toc_section = i;
225 else if (strcmp(secstrings+sechdrs[i].sh_name,"__versions")==0)
226 dedotify_versions((void *)hdr + sechdrs[i].sh_offset,
229 /* We don't handle .init for the moment: rename to _init */
230 while ((p = strstr(secstrings + sechdrs[i].sh_name, ".init")))
233 if (sechdrs[i].sh_type == SHT_SYMTAB)
234 dedotify((void *)hdr + sechdrs[i].sh_offset,
235 sechdrs[i].sh_size / sizeof(Elf64_Sym),
237 + sechdrs[sechdrs[i].sh_link].sh_offset);
240 if (!me->arch.stubs_section) {
241 printk("%s: doesn't contain .stubs.\n", me->name);
245 /* If we don't have a .toc, just use .stubs. We need to set r2
246 to some reasonable value in case the module calls out to
247 other functions via a stub, or if a function pointer escapes
248 the module by some means. */
249 if (!me->arch.toc_section)
250 me->arch.toc_section = me->arch.stubs_section;
252 /* Override the stubs size */
253 sechdrs[me->arch.stubs_section].sh_size = get_stubs_size(hdr, sechdrs);
257 /* r2 is the TOC pointer: it actually points 0x8000 into the TOC (this
258 gives the value maximum span in an instruction which uses a signed
260 static inline unsigned long my_r2(Elf64_Shdr *sechdrs, struct module *me)
262 return sechdrs[me->arch.toc_section].sh_addr + 0x8000;
265 /* Both low and high 16 bits are added as SIGNED additions, so if low
266 16 bits has high bit set, high 16 bits must be adjusted. These
267 macros do that (stolen from binutils). */
268 #define PPC_LO(v) ((v) & 0xffff)
269 #define PPC_HI(v) (((v) >> 16) & 0xffff)
270 #define PPC_HA(v) PPC_HI ((v) + 0x8000)
272 /* Patch stub to reference function and correct r2 value. */
273 static inline int create_stub(Elf64_Shdr *sechdrs,
274 struct ppc64_stub_entry *entry,
275 struct ppc64_opd_entry *opd,
278 Elf64_Half *loc1, *loc2;
283 #ifdef __LITTLE_ENDIAN__
284 loc1 = (Elf64_Half *)&entry->jump[0];
285 loc2 = (Elf64_Half *)&entry->jump[4];
287 loc1 = (Elf64_Half *)&entry->jump[2];
288 loc2 = (Elf64_Half *)&entry->jump[6];
291 /* Stub uses address relative to r2. */
292 reladdr = (unsigned long)entry - my_r2(sechdrs, me);
293 if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
294 printk("%s: Address %p of stub out of range of %p.\n",
295 me->name, (void *)reladdr, (void *)my_r2);
298 DEBUGP("Stub %p get data from reladdr %li\n", entry, reladdr);
300 *loc1 = PPC_HA(reladdr);
301 *loc2 = PPC_LO(reladdr);
302 entry->opd.funcaddr = opd->funcaddr;
303 entry->opd.r2 = opd->r2;
307 /* Create stub to jump to function described in this OPD: we need the
308 stub to set up the TOC ptr (r2) for the function. */
309 static unsigned long stub_for_addr(Elf64_Shdr *sechdrs,
310 unsigned long opdaddr,
313 struct ppc64_stub_entry *stubs;
314 struct ppc64_opd_entry *opd = (void *)opdaddr;
315 unsigned int i, num_stubs;
317 num_stubs = sechdrs[me->arch.stubs_section].sh_size / sizeof(*stubs);
319 /* Find this stub, or if that fails, the next avail. entry */
320 stubs = (void *)sechdrs[me->arch.stubs_section].sh_addr;
321 for (i = 0; stubs[i].opd.funcaddr; i++) {
322 BUG_ON(i >= num_stubs);
324 if (stubs[i].opd.funcaddr == opd->funcaddr)
325 return (unsigned long)&stubs[i];
328 if (!create_stub(sechdrs, &stubs[i], opd, me))
331 return (unsigned long)&stubs[i];
334 /* We expect a noop next: if it is, replace it with instruction to
336 static int restore_r2(u32 *instruction, struct module *me)
338 if (*instruction != PPC_INST_NOP) {
339 printk("%s: Expect noop after relocate, got %08x\n",
340 me->name, *instruction);
343 *instruction = 0xe8410028; /* ld r2,40(r1) */
347 int apply_relocate_add(Elf64_Shdr *sechdrs,
349 unsigned int symindex,
354 Elf64_Rela *rela = (void *)sechdrs[relsec].sh_addr;
356 unsigned long *location;
359 DEBUGP("Applying ADD relocate section %u to %u\n", relsec,
360 sechdrs[relsec].sh_info);
361 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rela); i++) {
362 /* This is where to make the change */
363 location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr
365 /* This is the symbol it is referring to */
366 sym = (Elf64_Sym *)sechdrs[symindex].sh_addr
367 + ELF64_R_SYM(rela[i].r_info);
369 DEBUGP("RELOC at %p: %li-type as %s (%lu) + %li\n",
370 location, (long)ELF64_R_TYPE(rela[i].r_info),
371 strtab + sym->st_name, (unsigned long)sym->st_value,
372 (long)rela[i].r_addend);
374 /* `Everything is relative'. */
375 value = sym->st_value + rela[i].r_addend;
377 switch (ELF64_R_TYPE(rela[i].r_info)) {
380 *(u32 *)location = value;
385 *(unsigned long *)location = value;
389 *(unsigned long *)location = my_r2(sechdrs, me);
393 /* Subtract TOC pointer */
394 value -= my_r2(sechdrs, me);
395 if (value + 0x8000 > 0xffff) {
396 printk("%s: bad TOC16 relocation (%lu)\n",
400 *((uint16_t *) location)
401 = (*((uint16_t *) location) & ~0xffff)
405 case R_PPC64_TOC16_LO:
406 /* Subtract TOC pointer */
407 value -= my_r2(sechdrs, me);
408 *((uint16_t *) location)
409 = (*((uint16_t *) location) & ~0xffff)
413 case R_PPC64_TOC16_DS:
414 /* Subtract TOC pointer */
415 value -= my_r2(sechdrs, me);
416 if ((value & 3) != 0 || value + 0x8000 > 0xffff) {
417 printk("%s: bad TOC16_DS relocation (%lu)\n",
421 *((uint16_t *) location)
422 = (*((uint16_t *) location) & ~0xfffc)
426 case R_PPC64_TOC16_LO_DS:
427 /* Subtract TOC pointer */
428 value -= my_r2(sechdrs, me);
429 if ((value & 3) != 0) {
430 printk("%s: bad TOC16_LO_DS relocation (%lu)\n",
434 *((uint16_t *) location)
435 = (*((uint16_t *) location) & ~0xfffc)
439 case R_PPC64_TOC16_HA:
440 /* Subtract TOC pointer */
441 value -= my_r2(sechdrs, me);
442 value = ((value + 0x8000) >> 16);
443 *((uint16_t *) location)
444 = (*((uint16_t *) location) & ~0xffff)
449 /* FIXME: Handle weak symbols here --RR */
450 if (sym->st_shndx == SHN_UNDEF) {
451 /* External: go via stub */
452 value = stub_for_addr(sechdrs, value, me);
455 if (!restore_r2((u32 *)location + 1, me))
459 /* Convert value to relative */
460 value -= (unsigned long)location;
461 if (value + 0x2000000 > 0x3ffffff || (value & 3) != 0){
462 printk("%s: REL24 %li out of range!\n",
463 me->name, (long int)value);
467 /* Only replace bits 2 through 26 */
468 *(uint32_t *)location
469 = (*(uint32_t *)location & ~0x03fffffc)
470 | (value & 0x03fffffc);
474 /* 64 bits relative (used by features fixups) */
475 *location = value - (unsigned long)location;
479 printk("%s: Unknown ADD relocation: %lu\n",
481 (unsigned long)ELF64_R_TYPE(rela[i].r_info));
486 #ifdef CONFIG_DYNAMIC_FTRACE
487 me->arch.toc = my_r2(sechdrs, me);
488 me->arch.tramp = stub_for_addr(sechdrs,
489 (unsigned long)ftrace_caller,