xref: /openbmc/linux/arch/ia64/kernel/module.c (revision ac3b4328)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * IA-64-specific support for kernel module loader.
4  *
5  * Copyright (C) 2003 Hewlett-Packard Co
6  *	David Mosberger-Tang <davidm@hpl.hp.com>
7  *
8  * Loosely based on patch by Rusty Russell.
9  */
10 
11 /* relocs tested so far:
12 
13    DIR64LSB
14    FPTR64LSB
15    GPREL22
16    LDXMOV
17    LDXMOV
18    LTOFF22
19    LTOFF22X
20    LTOFF22X
21    LTOFF_FPTR22
22    PCREL21B	(for br.call only; br.cond is not supported out of modules!)
23    PCREL60B	(for brl.cond only; brl.call is not supported for modules!)
24    PCREL64LSB
25    SECREL32LSB
26    SEGREL64LSB
27  */
28 
29 
30 #include <linux/kernel.h>
31 #include <linux/sched.h>
32 #include <linux/elf.h>
33 #include <linux/moduleloader.h>
34 #include <linux/string.h>
35 #include <linux/vmalloc.h>
36 
37 #include <asm/patch.h>
38 #include <asm/unaligned.h>
39 #include <asm/sections.h>
40 
41 #define ARCH_MODULE_DEBUG 0
42 
43 #if ARCH_MODULE_DEBUG
44 # define DEBUGP printk
45 # define inline
46 #else
47 # define DEBUGP(fmt , a...)
48 #endif
49 
50 #ifdef CONFIG_ITANIUM
51 # define USE_BRL	0
52 #else
53 # define USE_BRL	1
54 #endif
55 
56 #define MAX_LTOFF	((uint64_t) (1 << 22))	/* max. allowable linkage-table offset */
57 
58 /* Define some relocation helper macros/types: */
59 
60 #define FORMAT_SHIFT	0
61 #define FORMAT_BITS	3
62 #define FORMAT_MASK	((1 << FORMAT_BITS) - 1)
63 #define VALUE_SHIFT	3
64 #define VALUE_BITS	5
65 #define VALUE_MASK	((1 << VALUE_BITS) - 1)
66 
67 enum reloc_target_format {
68 	/* direct encoded formats: */
69 	RF_NONE = 0,
70 	RF_INSN14 = 1,
71 	RF_INSN22 = 2,
72 	RF_INSN64 = 3,
73 	RF_32MSB = 4,
74 	RF_32LSB = 5,
75 	RF_64MSB = 6,
76 	RF_64LSB = 7,
77 
78 	/* formats that cannot be directly decoded: */
79 	RF_INSN60,
80 	RF_INSN21B,	/* imm21 form 1 */
81 	RF_INSN21M,	/* imm21 form 2 */
82 	RF_INSN21F	/* imm21 form 3 */
83 };
84 
85 enum reloc_value_formula {
86 	RV_DIRECT = 4,		/* S + A */
87 	RV_GPREL = 5,		/* @gprel(S + A) */
88 	RV_LTREL = 6,		/* @ltoff(S + A) */
89 	RV_PLTREL = 7,		/* @pltoff(S + A) */
90 	RV_FPTR = 8,		/* @fptr(S + A) */
91 	RV_PCREL = 9,		/* S + A - P */
92 	RV_LTREL_FPTR = 10,	/* @ltoff(@fptr(S + A)) */
93 	RV_SEGREL = 11,		/* @segrel(S + A) */
94 	RV_SECREL = 12,		/* @secrel(S + A) */
95 	RV_BDREL = 13,		/* BD + A */
96 	RV_LTV = 14,		/* S + A (like RV_DIRECT, except frozen at static link-time) */
97 	RV_PCREL2 = 15,		/* S + A - P */
98 	RV_SPECIAL = 16,	/* various (see below) */
99 	RV_RSVD17 = 17,
100 	RV_TPREL = 18,		/* @tprel(S + A) */
101 	RV_LTREL_TPREL = 19,	/* @ltoff(@tprel(S + A)) */
102 	RV_DTPMOD = 20,		/* @dtpmod(S + A) */
103 	RV_LTREL_DTPMOD = 21,	/* @ltoff(@dtpmod(S + A)) */
104 	RV_DTPREL = 22,		/* @dtprel(S + A) */
105 	RV_LTREL_DTPREL = 23,	/* @ltoff(@dtprel(S + A)) */
106 	RV_RSVD24 = 24,
107 	RV_RSVD25 = 25,
108 	RV_RSVD26 = 26,
109 	RV_RSVD27 = 27
110 	/* 28-31 reserved for implementation-specific purposes.  */
111 };
112 
113 #define N(reloc)	[R_IA64_##reloc] = #reloc
114 
115 static const char *reloc_name[256] = {
116 	N(NONE),		N(IMM14),		N(IMM22),		N(IMM64),
117 	N(DIR32MSB),		N(DIR32LSB),		N(DIR64MSB),		N(DIR64LSB),
118 	N(GPREL22),		N(GPREL64I),		N(GPREL32MSB),		N(GPREL32LSB),
119 	N(GPREL64MSB),		N(GPREL64LSB),		N(LTOFF22),		N(LTOFF64I),
120 	N(PLTOFF22),		N(PLTOFF64I),		N(PLTOFF64MSB),		N(PLTOFF64LSB),
121 	N(FPTR64I),		N(FPTR32MSB),		N(FPTR32LSB),		N(FPTR64MSB),
122 	N(FPTR64LSB),		N(PCREL60B),		N(PCREL21B),		N(PCREL21M),
123 	N(PCREL21F),		N(PCREL32MSB),		N(PCREL32LSB),		N(PCREL64MSB),
124 	N(PCREL64LSB),		N(LTOFF_FPTR22),	N(LTOFF_FPTR64I),	N(LTOFF_FPTR32MSB),
125 	N(LTOFF_FPTR32LSB),	N(LTOFF_FPTR64MSB),	N(LTOFF_FPTR64LSB),	N(SEGREL32MSB),
126 	N(SEGREL32LSB),		N(SEGREL64MSB),		N(SEGREL64LSB),		N(SECREL32MSB),
127 	N(SECREL32LSB),		N(SECREL64MSB),		N(SECREL64LSB),		N(REL32MSB),
128 	N(REL32LSB),		N(REL64MSB),		N(REL64LSB),		N(LTV32MSB),
129 	N(LTV32LSB),		N(LTV64MSB),		N(LTV64LSB),		N(PCREL21BI),
130 	N(PCREL22),		N(PCREL64I),		N(IPLTMSB),		N(IPLTLSB),
131 	N(COPY),		N(LTOFF22X),		N(LDXMOV),		N(TPREL14),
132 	N(TPREL22),		N(TPREL64I),		N(TPREL64MSB),		N(TPREL64LSB),
133 	N(LTOFF_TPREL22),	N(DTPMOD64MSB),		N(DTPMOD64LSB),		N(LTOFF_DTPMOD22),
134 	N(DTPREL14),		N(DTPREL22),		N(DTPREL64I),		N(DTPREL32MSB),
135 	N(DTPREL32LSB),		N(DTPREL64MSB),		N(DTPREL64LSB),		N(LTOFF_DTPREL22)
136 };
137 
138 #undef N
139 
140 /* Opaque struct for insns, to protect against derefs. */
141 struct insn;
142 
143 static inline uint64_t
bundle(const struct insn * insn)144 bundle (const struct insn *insn)
145 {
146 	return (uint64_t) insn & ~0xfUL;
147 }
148 
149 static inline int
slot(const struct insn * insn)150 slot (const struct insn *insn)
151 {
152 	return (uint64_t) insn & 0x3;
153 }
154 
155 static int
apply_imm64(struct module * mod,struct insn * insn,uint64_t val)156 apply_imm64 (struct module *mod, struct insn *insn, uint64_t val)
157 {
158 	if (slot(insn) != 1 && slot(insn) != 2) {
159 		printk(KERN_ERR "%s: invalid slot number %d for IMM64\n",
160 		       mod->name, slot(insn));
161 		return 0;
162 	}
163 	ia64_patch_imm64((u64) insn, val);
164 	return 1;
165 }
166 
167 static int
apply_imm60(struct module * mod,struct insn * insn,uint64_t val)168 apply_imm60 (struct module *mod, struct insn *insn, uint64_t val)
169 {
170 	if (slot(insn) != 1 && slot(insn) != 2) {
171 		printk(KERN_ERR "%s: invalid slot number %d for IMM60\n",
172 		       mod->name, slot(insn));
173 		return 0;
174 	}
175 	if (val + ((uint64_t) 1 << 59) >= (1UL << 60)) {
176 		printk(KERN_ERR "%s: value %ld out of IMM60 range\n",
177 			mod->name, (long) val);
178 		return 0;
179 	}
180 	ia64_patch_imm60((u64) insn, val);
181 	return 1;
182 }
183 
184 static int
apply_imm22(struct module * mod,struct insn * insn,uint64_t val)185 apply_imm22 (struct module *mod, struct insn *insn, uint64_t val)
186 {
187 	if (val + (1 << 21) >= (1 << 22)) {
188 		printk(KERN_ERR "%s: value %li out of IMM22 range\n",
189 			mod->name, (long)val);
190 		return 0;
191 	}
192 	ia64_patch((u64) insn, 0x01fffcfe000UL, (  ((val & 0x200000UL) << 15) /* bit 21 -> 36 */
193 					         | ((val & 0x1f0000UL) <<  6) /* bit 16 -> 22 */
194 					         | ((val & 0x00ff80UL) << 20) /* bit  7 -> 27 */
195 					         | ((val & 0x00007fUL) << 13) /* bit  0 -> 13 */));
196 	return 1;
197 }
198 
199 static int
apply_imm21b(struct module * mod,struct insn * insn,uint64_t val)200 apply_imm21b (struct module *mod, struct insn *insn, uint64_t val)
201 {
202 	if (val + (1 << 20) >= (1 << 21)) {
203 		printk(KERN_ERR "%s: value %li out of IMM21b range\n",
204 			mod->name, (long)val);
205 		return 0;
206 	}
207 	ia64_patch((u64) insn, 0x11ffffe000UL, (  ((val & 0x100000UL) << 16) /* bit 20 -> 36 */
208 					        | ((val & 0x0fffffUL) << 13) /* bit  0 -> 13 */));
209 	return 1;
210 }
211 
212 #if USE_BRL
213 
214 struct plt_entry {
215 	/* Three instruction bundles in PLT. */
216  	unsigned char bundle[2][16];
217 };
218 
219 static const struct plt_entry ia64_plt_template = {
220 	{
221 		{
222 			0x04, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */
223 			0x00, 0x00, 0x00, 0x00, 0x00, 0x20, /*	     movl gp=TARGET_GP */
224 			0x00, 0x00, 0x00, 0x60
225 		},
226 		{
227 			0x05, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */
228 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*	     brl.many gp=TARGET_GP */
229 			0x08, 0x00, 0x00, 0xc0
230 		}
231 	}
232 };
233 
234 static int
patch_plt(struct module * mod,struct plt_entry * plt,long target_ip,unsigned long target_gp)235 patch_plt (struct module *mod, struct plt_entry *plt, long target_ip, unsigned long target_gp)
236 {
237 	if (apply_imm64(mod, (struct insn *) (plt->bundle[0] + 2), target_gp)
238 	    && apply_imm60(mod, (struct insn *) (plt->bundle[1] + 2),
239 			   (target_ip - (int64_t) plt->bundle[1]) / 16))
240 		return 1;
241 	return 0;
242 }
243 
244 unsigned long
plt_target(struct plt_entry * plt)245 plt_target (struct plt_entry *plt)
246 {
247 	uint64_t b0, b1, *b = (uint64_t *) plt->bundle[1];
248 	long off;
249 
250 	b0 = b[0]; b1 = b[1];
251 	off = (  ((b1 & 0x00fffff000000000UL) >> 36)		/* imm20b -> bit 0 */
252 	       | ((b0 >> 48) << 20) | ((b1 & 0x7fffffUL) << 36)	/* imm39 -> bit 20 */
253 	       | ((b1 & 0x0800000000000000UL) << 0));		/* i -> bit 59 */
254 	return (long) plt->bundle[1] + 16*off;
255 }
256 
257 #else /* !USE_BRL */
258 
259 struct plt_entry {
260 	/* Three instruction bundles in PLT. */
261  	unsigned char bundle[3][16];
262 };
263 
264 static const struct plt_entry ia64_plt_template = {
265 	{
266 		{
267 			0x05, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */
268 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*	     movl r16=TARGET_IP */
269 			0x02, 0x00, 0x00, 0x60
270 		},
271 		{
272 			0x04, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */
273 			0x00, 0x00, 0x00, 0x00, 0x00, 0x20, /*	     movl gp=TARGET_GP */
274 			0x00, 0x00, 0x00, 0x60
275 		},
276 		{
277 			0x11, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MIB] nop.m 0 */
278 			0x60, 0x80, 0x04, 0x80, 0x03, 0x00, /*	     mov b6=r16 */
279 			0x60, 0x00, 0x80, 0x00		    /*	     br.few b6 */
280 		}
281 	}
282 };
283 
284 static int
patch_plt(struct module * mod,struct plt_entry * plt,long target_ip,unsigned long target_gp)285 patch_plt (struct module *mod, struct plt_entry *plt, long target_ip, unsigned long target_gp)
286 {
287 	if (apply_imm64(mod, (struct insn *) (plt->bundle[0] + 2), target_ip)
288 	    && apply_imm64(mod, (struct insn *) (plt->bundle[1] + 2), target_gp))
289 		return 1;
290 	return 0;
291 }
292 
293 unsigned long
plt_target(struct plt_entry * plt)294 plt_target (struct plt_entry *plt)
295 {
296 	uint64_t b0, b1, *b = (uint64_t *) plt->bundle[0];
297 
298 	b0 = b[0]; b1 = b[1];
299 	return (  ((b1 & 0x000007f000000000) >> 36)		/* imm7b -> bit 0 */
300 		| ((b1 & 0x07fc000000000000) >> 43)		/* imm9d -> bit 7 */
301 		| ((b1 & 0x0003e00000000000) >> 29)		/* imm5c -> bit 16 */
302 		| ((b1 & 0x0000100000000000) >> 23)		/* ic -> bit 21 */
303 		| ((b0 >> 46) << 22) | ((b1 & 0x7fffff) << 40)	/* imm41 -> bit 22 */
304 		| ((b1 & 0x0800000000000000) <<  4));		/* i -> bit 63 */
305 }
306 
307 #endif /* !USE_BRL */
308 
309 void
module_arch_freeing_init(struct module * mod)310 module_arch_freeing_init (struct module *mod)
311 {
312 	if (mod->arch.init_unw_table) {
313 		unw_remove_unwind_table(mod->arch.init_unw_table);
314 		mod->arch.init_unw_table = NULL;
315 	}
316 }
317 
318 /* Have we already seen one of these relocations? */
319 /* FIXME: we could look in other sections, too --RR */
320 static int
duplicate_reloc(const Elf64_Rela * rela,unsigned int num)321 duplicate_reloc (const Elf64_Rela *rela, unsigned int num)
322 {
323 	unsigned int i;
324 
325 	for (i = 0; i < num; i++) {
326 		if (rela[i].r_info == rela[num].r_info && rela[i].r_addend == rela[num].r_addend)
327 			return 1;
328 	}
329 	return 0;
330 }
331 
332 /* Count how many GOT entries we may need */
333 static unsigned int
count_gots(const Elf64_Rela * rela,unsigned int num)334 count_gots (const Elf64_Rela *rela, unsigned int num)
335 {
336 	unsigned int i, ret = 0;
337 
338 	/* Sure, this is order(n^2), but it's usually short, and not
339            time critical */
340 	for (i = 0; i < num; i++) {
341 		switch (ELF64_R_TYPE(rela[i].r_info)) {
342 		      case R_IA64_LTOFF22:
343 		      case R_IA64_LTOFF22X:
344 		      case R_IA64_LTOFF64I:
345 		      case R_IA64_LTOFF_FPTR22:
346 		      case R_IA64_LTOFF_FPTR64I:
347 		      case R_IA64_LTOFF_FPTR32MSB:
348 		      case R_IA64_LTOFF_FPTR32LSB:
349 		      case R_IA64_LTOFF_FPTR64MSB:
350 		      case R_IA64_LTOFF_FPTR64LSB:
351 			if (!duplicate_reloc(rela, i))
352 				ret++;
353 			break;
354 		}
355 	}
356 	return ret;
357 }
358 
359 /* Count how many PLT entries we may need */
360 static unsigned int
count_plts(const Elf64_Rela * rela,unsigned int num)361 count_plts (const Elf64_Rela *rela, unsigned int num)
362 {
363 	unsigned int i, ret = 0;
364 
365 	/* Sure, this is order(n^2), but it's usually short, and not
366            time critical */
367 	for (i = 0; i < num; i++) {
368 		switch (ELF64_R_TYPE(rela[i].r_info)) {
369 		      case R_IA64_PCREL21B:
370 		      case R_IA64_PLTOFF22:
371 		      case R_IA64_PLTOFF64I:
372 		      case R_IA64_PLTOFF64MSB:
373 		      case R_IA64_PLTOFF64LSB:
374 		      case R_IA64_IPLTMSB:
375 		      case R_IA64_IPLTLSB:
376 			if (!duplicate_reloc(rela, i))
377 				ret++;
378 			break;
379 		}
380 	}
381 	return ret;
382 }
383 
384 /* We need to create an function-descriptors for any internal function
385    which is referenced. */
386 static unsigned int
count_fdescs(const Elf64_Rela * rela,unsigned int num)387 count_fdescs (const Elf64_Rela *rela, unsigned int num)
388 {
389 	unsigned int i, ret = 0;
390 
391 	/* Sure, this is order(n^2), but it's usually short, and not time critical.  */
392 	for (i = 0; i < num; i++) {
393 		switch (ELF64_R_TYPE(rela[i].r_info)) {
394 		      case R_IA64_FPTR64I:
395 		      case R_IA64_FPTR32LSB:
396 		      case R_IA64_FPTR32MSB:
397 		      case R_IA64_FPTR64LSB:
398 		      case R_IA64_FPTR64MSB:
399 		      case R_IA64_LTOFF_FPTR22:
400 		      case R_IA64_LTOFF_FPTR32LSB:
401 		      case R_IA64_LTOFF_FPTR32MSB:
402 		      case R_IA64_LTOFF_FPTR64I:
403 		      case R_IA64_LTOFF_FPTR64LSB:
404 		      case R_IA64_LTOFF_FPTR64MSB:
405 		      case R_IA64_IPLTMSB:
406 		      case R_IA64_IPLTLSB:
407 			/*
408 			 * Jumps to static functions sometimes go straight to their
409 			 * offset.  Of course, that may not be possible if the jump is
410 			 * from init -> core or vice. versa, so we need to generate an
411 			 * FDESC (and PLT etc) for that.
412 			 */
413 		      case R_IA64_PCREL21B:
414 			if (!duplicate_reloc(rela, i))
415 				ret++;
416 			break;
417 		}
418 	}
419 	return ret;
420 }
421 
422 int
module_frob_arch_sections(Elf_Ehdr * ehdr,Elf_Shdr * sechdrs,char * secstrings,struct module * mod)423 module_frob_arch_sections (Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, char *secstrings,
424 			   struct module *mod)
425 {
426 	unsigned long core_plts = 0, init_plts = 0, gots = 0, fdescs = 0;
427 	Elf64_Shdr *s, *sechdrs_end = sechdrs + ehdr->e_shnum;
428 
429 	/*
430 	 * To store the PLTs and function-descriptors, we expand the .text section for
431 	 * core module-code and the .init.text section for initialization code.
432 	 */
433 	for (s = sechdrs; s < sechdrs_end; ++s)
434 		if (strcmp(".core.plt", secstrings + s->sh_name) == 0)
435 			mod->arch.core_plt = s;
436 		else if (strcmp(".init.plt", secstrings + s->sh_name) == 0)
437 			mod->arch.init_plt = s;
438 		else if (strcmp(".got", secstrings + s->sh_name) == 0)
439 			mod->arch.got = s;
440 		else if (strcmp(".opd", secstrings + s->sh_name) == 0)
441 			mod->arch.opd = s;
442 		else if (strcmp(".IA_64.unwind", secstrings + s->sh_name) == 0)
443 			mod->arch.unwind = s;
444 
445 	if (!mod->arch.core_plt || !mod->arch.init_plt || !mod->arch.got || !mod->arch.opd) {
446 		printk(KERN_ERR "%s: sections missing\n", mod->name);
447 		return -ENOEXEC;
448 	}
449 
450 	/* GOT and PLTs can occur in any relocated section... */
451 	for (s = sechdrs + 1; s < sechdrs_end; ++s) {
452 		const Elf64_Rela *rels = (void *)ehdr + s->sh_offset;
453 		unsigned long numrels = s->sh_size/sizeof(Elf64_Rela);
454 
455 		if (s->sh_type != SHT_RELA)
456 			continue;
457 
458 		gots += count_gots(rels, numrels);
459 		fdescs += count_fdescs(rels, numrels);
460 		if (strstr(secstrings + s->sh_name, ".init"))
461 			init_plts += count_plts(rels, numrels);
462 		else
463 			core_plts += count_plts(rels, numrels);
464 	}
465 
466 	mod->arch.core_plt->sh_type = SHT_NOBITS;
467 	mod->arch.core_plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
468 	mod->arch.core_plt->sh_addralign = 16;
469 	mod->arch.core_plt->sh_size = core_plts * sizeof(struct plt_entry);
470 	mod->arch.init_plt->sh_type = SHT_NOBITS;
471 	mod->arch.init_plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
472 	mod->arch.init_plt->sh_addralign = 16;
473 	mod->arch.init_plt->sh_size = init_plts * sizeof(struct plt_entry);
474 	mod->arch.got->sh_type = SHT_NOBITS;
475 	mod->arch.got->sh_flags = ARCH_SHF_SMALL | SHF_ALLOC;
476 	mod->arch.got->sh_addralign = 8;
477 	mod->arch.got->sh_size = gots * sizeof(struct got_entry);
478 	mod->arch.opd->sh_type = SHT_NOBITS;
479 	mod->arch.opd->sh_flags = SHF_ALLOC;
480 	mod->arch.opd->sh_addralign = 8;
481 	mod->arch.opd->sh_size = fdescs * sizeof(struct fdesc);
482 	DEBUGP("%s: core.plt=%lx, init.plt=%lx, got=%lx, fdesc=%lx\n",
483 	       __func__, mod->arch.core_plt->sh_size, mod->arch.init_plt->sh_size,
484 	       mod->arch.got->sh_size, mod->arch.opd->sh_size);
485 	return 0;
486 }
487 
488 static inline bool
in_init(const struct module * mod,uint64_t addr)489 in_init (const struct module *mod, uint64_t addr)
490 {
491 	return within_module_init(addr, mod);
492 }
493 
494 static inline bool
in_core(const struct module * mod,uint64_t addr)495 in_core (const struct module *mod, uint64_t addr)
496 {
497 	return within_module_core(addr, mod);
498 }
499 
500 static inline bool
is_internal(const struct module * mod,uint64_t value)501 is_internal (const struct module *mod, uint64_t value)
502 {
503 	return in_init(mod, value) || in_core(mod, value);
504 }
505 
506 /*
507  * Get gp-relative offset for the linkage-table entry of VALUE.
508  */
509 static uint64_t
get_ltoff(struct module * mod,uint64_t value,int * okp)510 get_ltoff (struct module *mod, uint64_t value, int *okp)
511 {
512 	struct got_entry *got, *e;
513 
514 	if (!*okp)
515 		return 0;
516 
517 	got = (void *) mod->arch.got->sh_addr;
518 	for (e = got; e < got + mod->arch.next_got_entry; ++e)
519 		if (e->val == value)
520 			goto found;
521 
522 	/* Not enough GOT entries? */
523 	BUG_ON(e >= (struct got_entry *) (mod->arch.got->sh_addr + mod->arch.got->sh_size));
524 
525 	e->val = value;
526 	++mod->arch.next_got_entry;
527   found:
528 	return (uint64_t) e - mod->arch.gp;
529 }
530 
531 static inline int
gp_addressable(struct module * mod,uint64_t value)532 gp_addressable (struct module *mod, uint64_t value)
533 {
534 	return value - mod->arch.gp + MAX_LTOFF/2 < MAX_LTOFF;
535 }
536 
537 /* Get PC-relative PLT entry for this value.  Returns 0 on failure. */
538 static uint64_t
get_plt(struct module * mod,const struct insn * insn,uint64_t value,int * okp)539 get_plt (struct module *mod, const struct insn *insn, uint64_t value, int *okp)
540 {
541 	struct plt_entry *plt, *plt_end;
542 	uint64_t target_ip, target_gp;
543 
544 	if (!*okp)
545 		return 0;
546 
547 	if (in_init(mod, (uint64_t) insn)) {
548 		plt = (void *) mod->arch.init_plt->sh_addr;
549 		plt_end = (void *) plt + mod->arch.init_plt->sh_size;
550 	} else {
551 		plt = (void *) mod->arch.core_plt->sh_addr;
552 		plt_end = (void *) plt + mod->arch.core_plt->sh_size;
553 	}
554 
555 	/* "value" is a pointer to a function-descriptor; fetch the target ip/gp from it: */
556 	target_ip = ((uint64_t *) value)[0];
557 	target_gp = ((uint64_t *) value)[1];
558 
559 	/* Look for existing PLT entry. */
560 	while (plt->bundle[0][0]) {
561 		if (plt_target(plt) == target_ip)
562 			goto found;
563 		if (++plt >= plt_end)
564 			BUG();
565 	}
566 	*plt = ia64_plt_template;
567 	if (!patch_plt(mod, plt, target_ip, target_gp)) {
568 		*okp = 0;
569 		return 0;
570 	}
571 #if ARCH_MODULE_DEBUG
572 	if (plt_target(plt) != target_ip) {
573 		printk("%s: mistargeted PLT: wanted %lx, got %lx\n",
574 		       __func__, target_ip, plt_target(plt));
575 		*okp = 0;
576 		return 0;
577 	}
578 #endif
579   found:
580 	return (uint64_t) plt;
581 }
582 
583 /* Get function descriptor for VALUE. */
584 static uint64_t
get_fdesc(struct module * mod,uint64_t value,int * okp)585 get_fdesc (struct module *mod, uint64_t value, int *okp)
586 {
587 	struct fdesc *fdesc = (void *) mod->arch.opd->sh_addr;
588 
589 	if (!*okp)
590 		return 0;
591 
592 	if (!value) {
593 		printk(KERN_ERR "%s: fdesc for zero requested!\n", mod->name);
594 		return 0;
595 	}
596 
597 	if (!is_internal(mod, value))
598 		/*
599 		 * If it's not a module-local entry-point, "value" already points to a
600 		 * function-descriptor.
601 		 */
602 		return value;
603 
604 	/* Look for existing function descriptor. */
605 	while (fdesc->addr) {
606 		if (fdesc->addr == value)
607 			return (uint64_t)fdesc;
608 		if ((uint64_t) ++fdesc >= mod->arch.opd->sh_addr + mod->arch.opd->sh_size)
609 			BUG();
610 	}
611 
612 	/* Create new one */
613 	fdesc->addr = value;
614 	fdesc->gp = mod->arch.gp;
615 	return (uint64_t) fdesc;
616 }
617 
618 static inline int
do_reloc(struct module * mod,uint8_t r_type,Elf64_Sym * sym,uint64_t addend,Elf64_Shdr * sec,void * location)619 do_reloc (struct module *mod, uint8_t r_type, Elf64_Sym *sym, uint64_t addend,
620 	  Elf64_Shdr *sec, void *location)
621 {
622 	enum reloc_target_format format = (r_type >> FORMAT_SHIFT) & FORMAT_MASK;
623 	enum reloc_value_formula formula = (r_type >> VALUE_SHIFT) & VALUE_MASK;
624 	uint64_t val;
625 	int ok = 1;
626 
627 	val = sym->st_value + addend;
628 
629 	switch (formula) {
630 	      case RV_SEGREL:	/* segment base is arbitrarily chosen to be 0 for kernel modules */
631 	      case RV_DIRECT:
632 		break;
633 
634 	      case RV_GPREL:	  val -= mod->arch.gp; break;
635 	      case RV_LTREL:	  val = get_ltoff(mod, val, &ok); break;
636 	      case RV_PLTREL:	  val = get_plt(mod, location, val, &ok); break;
637 	      case RV_FPTR:	  val = get_fdesc(mod, val, &ok); break;
638 	      case RV_SECREL:	  val -= sec->sh_addr; break;
639 	      case RV_LTREL_FPTR: val = get_ltoff(mod, get_fdesc(mod, val, &ok), &ok); break;
640 
641 	      case RV_PCREL:
642 		switch (r_type) {
643 		      case R_IA64_PCREL21B:
644 			if ((in_init(mod, val) && in_core(mod, (uint64_t)location)) ||
645 			    (in_core(mod, val) && in_init(mod, (uint64_t)location))) {
646 				/*
647 				 * Init section may have been allocated far away from core,
648 				 * if the branch won't reach, then allocate a plt for it.
649 				 */
650 				uint64_t delta = ((int64_t)val - (int64_t)location) / 16;
651 				if (delta + (1 << 20) >= (1 << 21)) {
652 					val = get_fdesc(mod, val, &ok);
653 					val = get_plt(mod, location, val, &ok);
654 				}
655 			} else if (!is_internal(mod, val))
656 				val = get_plt(mod, location, val, &ok);
657 			fallthrough;
658 		      default:
659 			val -= bundle(location);
660 			break;
661 
662 		      case R_IA64_PCREL32MSB:
663 		      case R_IA64_PCREL32LSB:
664 		      case R_IA64_PCREL64MSB:
665 		      case R_IA64_PCREL64LSB:
666 			val -= (uint64_t) location;
667 			break;
668 
669 		}
670 		switch (r_type) {
671 		      case R_IA64_PCREL60B: format = RF_INSN60; break;
672 		      case R_IA64_PCREL21B: format = RF_INSN21B; break;
673 		      case R_IA64_PCREL21M: format = RF_INSN21M; break;
674 		      case R_IA64_PCREL21F: format = RF_INSN21F; break;
675 		      default: break;
676 		}
677 		break;
678 
679 	      case RV_BDREL:
680 		val -= (uint64_t) (in_init(mod, val) ? mod->mem[MOD_INIT_TEXT].base :
681 				   mod->mem[MOD_TEXT].base);
682 		break;
683 
684 	      case RV_LTV:
685 		/* can link-time value relocs happen here?  */
686 		BUG();
687 		break;
688 
689 	      case RV_PCREL2:
690 		if (r_type == R_IA64_PCREL21BI) {
691 			if (!is_internal(mod, val)) {
692 				printk(KERN_ERR "%s: %s reloc against "
693 					"non-local symbol (%lx)\n", __func__,
694 					reloc_name[r_type], (unsigned long)val);
695 				return -ENOEXEC;
696 			}
697 			format = RF_INSN21B;
698 		}
699 		val -= bundle(location);
700 		break;
701 
702 	      case RV_SPECIAL:
703 		switch (r_type) {
704 		      case R_IA64_IPLTMSB:
705 		      case R_IA64_IPLTLSB:
706 			val = get_fdesc(mod, get_plt(mod, location, val, &ok), &ok);
707 			format = RF_64LSB;
708 			if (r_type == R_IA64_IPLTMSB)
709 				format = RF_64MSB;
710 			break;
711 
712 		      case R_IA64_SUB:
713 			val = addend - sym->st_value;
714 			format = RF_INSN64;
715 			break;
716 
717 		      case R_IA64_LTOFF22X:
718 			if (gp_addressable(mod, val))
719 				val -= mod->arch.gp;
720 			else
721 				val = get_ltoff(mod, val, &ok);
722 			format = RF_INSN22;
723 			break;
724 
725 		      case R_IA64_LDXMOV:
726 			if (gp_addressable(mod, val)) {
727 				/* turn "ld8" into "mov": */
728 				DEBUGP("%s: patching ld8 at %p to mov\n", __func__, location);
729 				ia64_patch((u64) location, 0x1fff80fe000UL, 0x10000000000UL);
730 			}
731 			return 0;
732 
733 		      default:
734 			if (reloc_name[r_type])
735 				printk(KERN_ERR "%s: special reloc %s not supported",
736 				       mod->name, reloc_name[r_type]);
737 			else
738 				printk(KERN_ERR "%s: unknown special reloc %x\n",
739 				       mod->name, r_type);
740 			return -ENOEXEC;
741 		}
742 		break;
743 
744 	      case RV_TPREL:
745 	      case RV_LTREL_TPREL:
746 	      case RV_DTPMOD:
747 	      case RV_LTREL_DTPMOD:
748 	      case RV_DTPREL:
749 	      case RV_LTREL_DTPREL:
750 		printk(KERN_ERR "%s: %s reloc not supported\n",
751 		       mod->name, reloc_name[r_type] ? reloc_name[r_type] : "?");
752 		return -ENOEXEC;
753 
754 	      default:
755 		printk(KERN_ERR "%s: unknown reloc %x\n", mod->name, r_type);
756 		return -ENOEXEC;
757 	}
758 
759 	if (!ok)
760 		return -ENOEXEC;
761 
762 	DEBUGP("%s: [%p]<-%016lx = %s(%lx)\n", __func__, location, val,
763 	       reloc_name[r_type] ? reloc_name[r_type] : "?", sym->st_value + addend);
764 
765 	switch (format) {
766 	      case RF_INSN21B:	ok = apply_imm21b(mod, location, (int64_t) val / 16); break;
767 	      case RF_INSN22:	ok = apply_imm22(mod, location, val); break;
768 	      case RF_INSN64:	ok = apply_imm64(mod, location, val); break;
769 	      case RF_INSN60:	ok = apply_imm60(mod, location, (int64_t) val / 16); break;
770 	      case RF_32LSB:	put_unaligned(val, (uint32_t *) location); break;
771 	      case RF_64LSB:	put_unaligned(val, (uint64_t *) location); break;
772 	      case RF_32MSB:	/* ia64 Linux is little-endian... */
773 	      case RF_64MSB:	/* ia64 Linux is little-endian... */
774 	      case RF_INSN14:	/* must be within-module, i.e., resolved by "ld -r" */
775 	      case RF_INSN21M:	/* must be within-module, i.e., resolved by "ld -r" */
776 	      case RF_INSN21F:	/* must be within-module, i.e., resolved by "ld -r" */
777 		printk(KERN_ERR "%s: format %u needed by %s reloc is not supported\n",
778 		       mod->name, format, reloc_name[r_type] ? reloc_name[r_type] : "?");
779 		return -ENOEXEC;
780 
781 	      default:
782 		printk(KERN_ERR "%s: relocation %s resulted in unknown format %u\n",
783 		       mod->name, reloc_name[r_type] ? reloc_name[r_type] : "?", format);
784 		return -ENOEXEC;
785 	}
786 	return ok ? 0 : -ENOEXEC;
787 }
788 
789 int
apply_relocate_add(Elf64_Shdr * sechdrs,const char * strtab,unsigned int symindex,unsigned int relsec,struct module * mod)790 apply_relocate_add (Elf64_Shdr *sechdrs, const char *strtab, unsigned int symindex,
791 		    unsigned int relsec, struct module *mod)
792 {
793 	unsigned int i, n = sechdrs[relsec].sh_size / sizeof(Elf64_Rela);
794 	Elf64_Rela *rela = (void *) sechdrs[relsec].sh_addr;
795 	Elf64_Shdr *target_sec;
796 	int ret;
797 
798 	DEBUGP("%s: applying section %u (%u relocs) to %u\n", __func__,
799 	       relsec, n, sechdrs[relsec].sh_info);
800 
801 	target_sec = sechdrs + sechdrs[relsec].sh_info;
802 
803 	if (target_sec->sh_entsize == ~0UL)
804 		/*
805 		 * If target section wasn't allocated, we don't need to relocate it.
806 		 * Happens, e.g., for debug sections.
807 		 */
808 		return 0;
809 
810 	if (!mod->arch.gp) {
811 		/*
812 		 * XXX Should have an arch-hook for running this after final section
813 		 *     addresses have been selected...
814 		 */
815 		uint64_t gp;
816 		struct module_memory *mod_mem;
817 
818 		mod_mem = &mod->mem[MOD_DATA];
819 		if (mod_mem->size > MAX_LTOFF)
820 			/*
821 			 * This takes advantage of fact that SHF_ARCH_SMALL gets allocated
822 			 * at the end of the module.
823 			 */
824 			gp = mod_mem->size - MAX_LTOFF / 2;
825 		else
826 			gp = mod_mem->size / 2;
827 		gp = (uint64_t) mod_mem->base + ((gp + 7) & -8);
828 		mod->arch.gp = gp;
829 		DEBUGP("%s: placing gp at 0x%lx\n", __func__, gp);
830 	}
831 
832 	for (i = 0; i < n; i++) {
833 		ret = do_reloc(mod, ELF64_R_TYPE(rela[i].r_info),
834 			       ((Elf64_Sym *) sechdrs[symindex].sh_addr
835 				+ ELF64_R_SYM(rela[i].r_info)),
836 			       rela[i].r_addend, target_sec,
837 			       (void *) target_sec->sh_addr + rela[i].r_offset);
838 		if (ret < 0)
839 			return ret;
840 	}
841 	return 0;
842 }
843 
844 /*
845  * Modules contain a single unwind table which covers both the core and the init text
846  * sections but since the two are not contiguous, we need to split this table up such that
847  * we can register (and unregister) each "segment" separately.  Fortunately, this sounds
848  * more complicated than it really is.
849  */
850 static void
register_unwind_table(struct module * mod)851 register_unwind_table (struct module *mod)
852 {
853 	struct unw_table_entry *start = (void *) mod->arch.unwind->sh_addr;
854 	struct unw_table_entry *end = start + mod->arch.unwind->sh_size / sizeof (*start);
855 	struct unw_table_entry *e1, *e2, *core, *init;
856 	unsigned long num_init = 0, num_core = 0;
857 
858 	/* First, count how many init and core unwind-table entries there are.  */
859 	for (e1 = start; e1 < end; ++e1)
860 		if (in_init(mod, e1->start_offset))
861 			++num_init;
862 		else
863 			++num_core;
864 	/*
865 	 * Second, sort the table such that all unwind-table entries for the init and core
866 	 * text sections are nicely separated.  We do this with a stupid bubble sort
867 	 * (unwind tables don't get ridiculously huge).
868 	 */
869 	for (e1 = start; e1 < end; ++e1) {
870 		for (e2 = e1 + 1; e2 < end; ++e2) {
871 			if (e2->start_offset < e1->start_offset) {
872 				swap(*e1, *e2);
873 			}
874 		}
875 	}
876 	/*
877 	 * Third, locate the init and core segments in the unwind table:
878 	 */
879 	if (in_init(mod, start->start_offset)) {
880 		init = start;
881 		core = start + num_init;
882 	} else {
883 		core = start;
884 		init = start + num_core;
885 	}
886 
887 	DEBUGP("%s: name=%s, gp=%lx, num_init=%lu, num_core=%lu\n", __func__,
888 	       mod->name, mod->arch.gp, num_init, num_core);
889 
890 	/*
891 	 * Fourth, register both tables (if not empty).
892 	 */
893 	if (num_core > 0) {
894 		mod->arch.core_unw_table = unw_add_unwind_table(mod->name, 0, mod->arch.gp,
895 								core, core + num_core);
896 		DEBUGP("%s:  core: handle=%p [%p-%p)\n", __func__,
897 		       mod->arch.core_unw_table, core, core + num_core);
898 	}
899 	if (num_init > 0) {
900 		mod->arch.init_unw_table = unw_add_unwind_table(mod->name, 0, mod->arch.gp,
901 								init, init + num_init);
902 		DEBUGP("%s:  init: handle=%p [%p-%p)\n", __func__,
903 		       mod->arch.init_unw_table, init, init + num_init);
904 	}
905 }
906 
907 int
module_finalize(const Elf_Ehdr * hdr,const Elf_Shdr * sechdrs,struct module * mod)908 module_finalize (const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *mod)
909 {
910 	struct mod_arch_specific *mas = &mod->arch;
911 
912 	DEBUGP("%s: init: entry=%p\n", __func__, mod->init);
913 	if (mas->unwind)
914 		register_unwind_table(mod);
915 
916 	/*
917 	 * ".opd" was already relocated to the final destination. Store
918 	 * it's address for use in symbolizer.
919 	 */
920 	mas->opd_addr = (void *)mas->opd->sh_addr;
921 	mas->opd_size = mas->opd->sh_size;
922 
923 	/*
924 	 * Module relocation was already done at this point. Section
925 	 * headers are about to be deleted. Wipe out load-time context.
926 	 */
927 	mas->core_plt = NULL;
928 	mas->init_plt = NULL;
929 	mas->got = NULL;
930 	mas->opd = NULL;
931 	mas->unwind = NULL;
932 	mas->gp = 0;
933 	mas->next_got_entry = 0;
934 
935 	return 0;
936 }
937 
938 void
module_arch_cleanup(struct module * mod)939 module_arch_cleanup (struct module *mod)
940 {
941 	if (mod->arch.init_unw_table) {
942 		unw_remove_unwind_table(mod->arch.init_unw_table);
943 		mod->arch.init_unw_table = NULL;
944 	}
945 	if (mod->arch.core_unw_table) {
946 		unw_remove_unwind_table(mod->arch.core_unw_table);
947 		mod->arch.core_unw_table = NULL;
948 	}
949 }
950 
dereference_module_function_descriptor(struct module * mod,void * ptr)951 void *dereference_module_function_descriptor(struct module *mod, void *ptr)
952 {
953 	struct mod_arch_specific *mas = &mod->arch;
954 
955 	if (ptr < mas->opd_addr || ptr >= mas->opd_addr + mas->opd_size)
956 		return ptr;
957 
958 	return dereference_function_descriptor(ptr);
959 }
960