1fcdfe9d2SHuacai Chen // SPDX-License-Identifier: GPL-2.0+
2fcdfe9d2SHuacai Chen /*
3fcdfe9d2SHuacai Chen * Author: Hanlu Li <lihanlu@loongson.cn>
4fcdfe9d2SHuacai Chen * Huacai Chen <chenhuacai@loongson.cn>
5fcdfe9d2SHuacai Chen *
6fcdfe9d2SHuacai Chen * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
7fcdfe9d2SHuacai Chen */
8fcdfe9d2SHuacai Chen
9fcdfe9d2SHuacai Chen #define pr_fmt(fmt) "kmod: " fmt
10fcdfe9d2SHuacai Chen
11fcdfe9d2SHuacai Chen #include <linux/moduleloader.h>
12fcdfe9d2SHuacai Chen #include <linux/elf.h>
13fcdfe9d2SHuacai Chen #include <linux/mm.h>
14d4b6f156SHuacai Chen #include <linux/numa.h>
15fcdfe9d2SHuacai Chen #include <linux/vmalloc.h>
16fcdfe9d2SHuacai Chen #include <linux/slab.h>
17fcdfe9d2SHuacai Chen #include <linux/fs.h>
1828ac0a9eSQing Zhang #include <linux/ftrace.h>
19fcdfe9d2SHuacai Chen #include <linux/string.h>
20fcdfe9d2SHuacai Chen #include <linux/kernel.h>
2119e5eb15SHuacai Chen #include <asm/alternative.h>
2228ac0a9eSQing Zhang #include <asm/inst.h>
23fcdfe9d2SHuacai Chen
rela_stack_push(s64 stack_value,s64 * rela_stack,size_t * rela_stack_top)24fcdfe9d2SHuacai Chen static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top)
25fcdfe9d2SHuacai Chen {
26fcdfe9d2SHuacai Chen if (*rela_stack_top >= RELA_STACK_DEPTH)
27fcdfe9d2SHuacai Chen return -ENOEXEC;
28fcdfe9d2SHuacai Chen
29fcdfe9d2SHuacai Chen rela_stack[(*rela_stack_top)++] = stack_value;
30fcdfe9d2SHuacai Chen pr_debug("%s stack_value = 0x%llx\n", __func__, stack_value);
31fcdfe9d2SHuacai Chen
32fcdfe9d2SHuacai Chen return 0;
33fcdfe9d2SHuacai Chen }
34fcdfe9d2SHuacai Chen
rela_stack_pop(s64 * stack_value,s64 * rela_stack,size_t * rela_stack_top)35fcdfe9d2SHuacai Chen static int rela_stack_pop(s64 *stack_value, s64 *rela_stack, size_t *rela_stack_top)
36fcdfe9d2SHuacai Chen {
37fcdfe9d2SHuacai Chen if (*rela_stack_top == 0)
38fcdfe9d2SHuacai Chen return -ENOEXEC;
39fcdfe9d2SHuacai Chen
40fcdfe9d2SHuacai Chen *stack_value = rela_stack[--(*rela_stack_top)];
41fcdfe9d2SHuacai Chen pr_debug("%s stack_value = 0x%llx\n", __func__, *stack_value);
42fcdfe9d2SHuacai Chen
43fcdfe9d2SHuacai Chen return 0;
44fcdfe9d2SHuacai Chen }
45fcdfe9d2SHuacai Chen
apply_r_larch_none(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)46fcdfe9d2SHuacai Chen static int apply_r_larch_none(struct module *mod, u32 *location, Elf_Addr v,
47fcdfe9d2SHuacai Chen s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
48fcdfe9d2SHuacai Chen {
49fcdfe9d2SHuacai Chen return 0;
50fcdfe9d2SHuacai Chen }
51fcdfe9d2SHuacai Chen
apply_r_larch_error(struct module * me,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)52fcdfe9d2SHuacai Chen static int apply_r_larch_error(struct module *me, u32 *location, Elf_Addr v,
53fcdfe9d2SHuacai Chen s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
54fcdfe9d2SHuacai Chen {
55fcdfe9d2SHuacai Chen pr_err("%s: Unsupport relocation type %u, please add its support.\n", me->name, type);
56fcdfe9d2SHuacai Chen return -EINVAL;
57fcdfe9d2SHuacai Chen }
58fcdfe9d2SHuacai Chen
apply_r_larch_32(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)59fcdfe9d2SHuacai Chen static int apply_r_larch_32(struct module *mod, u32 *location, Elf_Addr v,
60fcdfe9d2SHuacai Chen s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
61fcdfe9d2SHuacai Chen {
62fcdfe9d2SHuacai Chen *location = v;
63fcdfe9d2SHuacai Chen return 0;
64fcdfe9d2SHuacai Chen }
65fcdfe9d2SHuacai Chen
apply_r_larch_64(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)66fcdfe9d2SHuacai Chen static int apply_r_larch_64(struct module *mod, u32 *location, Elf_Addr v,
67fcdfe9d2SHuacai Chen s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
68fcdfe9d2SHuacai Chen {
69fcdfe9d2SHuacai Chen *(Elf_Addr *)location = v;
70fcdfe9d2SHuacai Chen return 0;
71fcdfe9d2SHuacai Chen }
72fcdfe9d2SHuacai Chen
apply_r_larch_sop_push_pcrel(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)73fcdfe9d2SHuacai Chen static int apply_r_larch_sop_push_pcrel(struct module *mod, u32 *location, Elf_Addr v,
74fcdfe9d2SHuacai Chen s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
75fcdfe9d2SHuacai Chen {
76fcdfe9d2SHuacai Chen return rela_stack_push(v - (u64)location, rela_stack, rela_stack_top);
77fcdfe9d2SHuacai Chen }
78fcdfe9d2SHuacai Chen
apply_r_larch_sop_push_absolute(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)79fcdfe9d2SHuacai Chen static int apply_r_larch_sop_push_absolute(struct module *mod, u32 *location, Elf_Addr v,
80fcdfe9d2SHuacai Chen s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
81fcdfe9d2SHuacai Chen {
82fcdfe9d2SHuacai Chen return rela_stack_push(v, rela_stack, rela_stack_top);
83fcdfe9d2SHuacai Chen }
84fcdfe9d2SHuacai Chen
apply_r_larch_sop_push_dup(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)85fcdfe9d2SHuacai Chen static int apply_r_larch_sop_push_dup(struct module *mod, u32 *location, Elf_Addr v,
86fcdfe9d2SHuacai Chen s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
87fcdfe9d2SHuacai Chen {
88fcdfe9d2SHuacai Chen int err = 0;
89fcdfe9d2SHuacai Chen s64 opr1;
90fcdfe9d2SHuacai Chen
91fcdfe9d2SHuacai Chen err = rela_stack_pop(&opr1, rela_stack, rela_stack_top);
92fcdfe9d2SHuacai Chen if (err)
93fcdfe9d2SHuacai Chen return err;
94fcdfe9d2SHuacai Chen err = rela_stack_push(opr1, rela_stack, rela_stack_top);
95fcdfe9d2SHuacai Chen if (err)
96fcdfe9d2SHuacai Chen return err;
97fcdfe9d2SHuacai Chen err = rela_stack_push(opr1, rela_stack, rela_stack_top);
98fcdfe9d2SHuacai Chen if (err)
99fcdfe9d2SHuacai Chen return err;
100fcdfe9d2SHuacai Chen
101fcdfe9d2SHuacai Chen return 0;
102fcdfe9d2SHuacai Chen }
103fcdfe9d2SHuacai Chen
apply_r_larch_sop_push_plt_pcrel(struct module * mod,Elf_Shdr * sechdrs,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)1049151dde4SHuacai Chen static int apply_r_larch_sop_push_plt_pcrel(struct module *mod,
1059151dde4SHuacai Chen Elf_Shdr *sechdrs, u32 *location, Elf_Addr v,
106fcdfe9d2SHuacai Chen s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
107fcdfe9d2SHuacai Chen {
108fcdfe9d2SHuacai Chen ptrdiff_t offset = (void *)v - (void *)location;
109fcdfe9d2SHuacai Chen
110fcdfe9d2SHuacai Chen if (offset >= SZ_128M)
1119151dde4SHuacai Chen v = module_emit_plt_entry(mod, sechdrs, v);
112fcdfe9d2SHuacai Chen
113fcdfe9d2SHuacai Chen if (offset < -SZ_128M)
1149151dde4SHuacai Chen v = module_emit_plt_entry(mod, sechdrs, v);
115fcdfe9d2SHuacai Chen
116fcdfe9d2SHuacai Chen return apply_r_larch_sop_push_pcrel(mod, location, v, rela_stack, rela_stack_top, type);
117fcdfe9d2SHuacai Chen }
118fcdfe9d2SHuacai Chen
apply_r_larch_sop(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)119fcdfe9d2SHuacai Chen static int apply_r_larch_sop(struct module *mod, u32 *location, Elf_Addr v,
120fcdfe9d2SHuacai Chen s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
121fcdfe9d2SHuacai Chen {
122fcdfe9d2SHuacai Chen int err = 0;
123fcdfe9d2SHuacai Chen s64 opr1, opr2, opr3;
124fcdfe9d2SHuacai Chen
125fcdfe9d2SHuacai Chen if (type == R_LARCH_SOP_IF_ELSE) {
126fcdfe9d2SHuacai Chen err = rela_stack_pop(&opr3, rela_stack, rela_stack_top);
127fcdfe9d2SHuacai Chen if (err)
128fcdfe9d2SHuacai Chen return err;
129fcdfe9d2SHuacai Chen }
130fcdfe9d2SHuacai Chen
131fcdfe9d2SHuacai Chen err = rela_stack_pop(&opr2, rela_stack, rela_stack_top);
132fcdfe9d2SHuacai Chen if (err)
133fcdfe9d2SHuacai Chen return err;
134fcdfe9d2SHuacai Chen err = rela_stack_pop(&opr1, rela_stack, rela_stack_top);
135fcdfe9d2SHuacai Chen if (err)
136fcdfe9d2SHuacai Chen return err;
137fcdfe9d2SHuacai Chen
138fcdfe9d2SHuacai Chen switch (type) {
139fcdfe9d2SHuacai Chen case R_LARCH_SOP_AND:
140fcdfe9d2SHuacai Chen err = rela_stack_push(opr1 & opr2, rela_stack, rela_stack_top);
141fcdfe9d2SHuacai Chen break;
142fcdfe9d2SHuacai Chen case R_LARCH_SOP_ADD:
143fcdfe9d2SHuacai Chen err = rela_stack_push(opr1 + opr2, rela_stack, rela_stack_top);
144fcdfe9d2SHuacai Chen break;
145fcdfe9d2SHuacai Chen case R_LARCH_SOP_SUB:
146fcdfe9d2SHuacai Chen err = rela_stack_push(opr1 - opr2, rela_stack, rela_stack_top);
147fcdfe9d2SHuacai Chen break;
148fcdfe9d2SHuacai Chen case R_LARCH_SOP_SL:
149fcdfe9d2SHuacai Chen err = rela_stack_push(opr1 << opr2, rela_stack, rela_stack_top);
150fcdfe9d2SHuacai Chen break;
151fcdfe9d2SHuacai Chen case R_LARCH_SOP_SR:
152fcdfe9d2SHuacai Chen err = rela_stack_push(opr1 >> opr2, rela_stack, rela_stack_top);
153fcdfe9d2SHuacai Chen break;
154fcdfe9d2SHuacai Chen case R_LARCH_SOP_IF_ELSE:
155fcdfe9d2SHuacai Chen err = rela_stack_push(opr1 ? opr2 : opr3, rela_stack, rela_stack_top);
156fcdfe9d2SHuacai Chen break;
157fcdfe9d2SHuacai Chen default:
158fcdfe9d2SHuacai Chen pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
159fcdfe9d2SHuacai Chen return -EINVAL;
160fcdfe9d2SHuacai Chen }
161fcdfe9d2SHuacai Chen
162fcdfe9d2SHuacai Chen return err;
163fcdfe9d2SHuacai Chen }
164fcdfe9d2SHuacai Chen
apply_r_larch_sop_imm_field(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)165fcdfe9d2SHuacai Chen static int apply_r_larch_sop_imm_field(struct module *mod, u32 *location, Elf_Addr v,
166fcdfe9d2SHuacai Chen s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
167fcdfe9d2SHuacai Chen {
168fcdfe9d2SHuacai Chen int err = 0;
169fcdfe9d2SHuacai Chen s64 opr1;
170fcdfe9d2SHuacai Chen union loongarch_instruction *insn = (union loongarch_instruction *)location;
171fcdfe9d2SHuacai Chen
172fcdfe9d2SHuacai Chen err = rela_stack_pop(&opr1, rela_stack, rela_stack_top);
173fcdfe9d2SHuacai Chen if (err)
174fcdfe9d2SHuacai Chen return err;
175fcdfe9d2SHuacai Chen
176fcdfe9d2SHuacai Chen switch (type) {
177fcdfe9d2SHuacai Chen case R_LARCH_SOP_POP_32_U_10_12:
178fcdfe9d2SHuacai Chen if (!unsigned_imm_check(opr1, 12))
179fcdfe9d2SHuacai Chen goto overflow;
180fcdfe9d2SHuacai Chen
181fcdfe9d2SHuacai Chen /* (*(uint32_t *) PC) [21 ... 10] = opr [11 ... 0] */
182fcdfe9d2SHuacai Chen insn->reg2i12_format.immediate = opr1 & 0xfff;
183fcdfe9d2SHuacai Chen return 0;
184fcdfe9d2SHuacai Chen case R_LARCH_SOP_POP_32_S_10_12:
185fcdfe9d2SHuacai Chen if (!signed_imm_check(opr1, 12))
186fcdfe9d2SHuacai Chen goto overflow;
187fcdfe9d2SHuacai Chen
188fcdfe9d2SHuacai Chen insn->reg2i12_format.immediate = opr1 & 0xfff;
189fcdfe9d2SHuacai Chen return 0;
190fcdfe9d2SHuacai Chen case R_LARCH_SOP_POP_32_S_10_16:
191fcdfe9d2SHuacai Chen if (!signed_imm_check(opr1, 16))
192fcdfe9d2SHuacai Chen goto overflow;
193fcdfe9d2SHuacai Chen
194fcdfe9d2SHuacai Chen insn->reg2i16_format.immediate = opr1 & 0xffff;
195fcdfe9d2SHuacai Chen return 0;
196fcdfe9d2SHuacai Chen case R_LARCH_SOP_POP_32_S_10_16_S2:
197fcdfe9d2SHuacai Chen if (opr1 % 4)
198fcdfe9d2SHuacai Chen goto unaligned;
199fcdfe9d2SHuacai Chen
200fcdfe9d2SHuacai Chen if (!signed_imm_check(opr1, 18))
201fcdfe9d2SHuacai Chen goto overflow;
202fcdfe9d2SHuacai Chen
203fcdfe9d2SHuacai Chen insn->reg2i16_format.immediate = (opr1 >> 2) & 0xffff;
204fcdfe9d2SHuacai Chen return 0;
205fcdfe9d2SHuacai Chen case R_LARCH_SOP_POP_32_S_5_20:
206fcdfe9d2SHuacai Chen if (!signed_imm_check(opr1, 20))
207fcdfe9d2SHuacai Chen goto overflow;
208fcdfe9d2SHuacai Chen
209fcdfe9d2SHuacai Chen insn->reg1i20_format.immediate = (opr1) & 0xfffff;
210fcdfe9d2SHuacai Chen return 0;
211fcdfe9d2SHuacai Chen case R_LARCH_SOP_POP_32_S_0_5_10_16_S2:
212fcdfe9d2SHuacai Chen if (opr1 % 4)
213fcdfe9d2SHuacai Chen goto unaligned;
214fcdfe9d2SHuacai Chen
215fcdfe9d2SHuacai Chen if (!signed_imm_check(opr1, 23))
216fcdfe9d2SHuacai Chen goto overflow;
217fcdfe9d2SHuacai Chen
218fcdfe9d2SHuacai Chen opr1 >>= 2;
219fcdfe9d2SHuacai Chen insn->reg1i21_format.immediate_l = opr1 & 0xffff;
220fcdfe9d2SHuacai Chen insn->reg1i21_format.immediate_h = (opr1 >> 16) & 0x1f;
221fcdfe9d2SHuacai Chen return 0;
222fcdfe9d2SHuacai Chen case R_LARCH_SOP_POP_32_S_0_10_10_16_S2:
223fcdfe9d2SHuacai Chen if (opr1 % 4)
224fcdfe9d2SHuacai Chen goto unaligned;
225fcdfe9d2SHuacai Chen
226fcdfe9d2SHuacai Chen if (!signed_imm_check(opr1, 28))
227fcdfe9d2SHuacai Chen goto overflow;
228fcdfe9d2SHuacai Chen
229fcdfe9d2SHuacai Chen opr1 >>= 2;
230fcdfe9d2SHuacai Chen insn->reg0i26_format.immediate_l = opr1 & 0xffff;
231fcdfe9d2SHuacai Chen insn->reg0i26_format.immediate_h = (opr1 >> 16) & 0x3ff;
232fcdfe9d2SHuacai Chen return 0;
233fcdfe9d2SHuacai Chen case R_LARCH_SOP_POP_32_U:
234fcdfe9d2SHuacai Chen if (!unsigned_imm_check(opr1, 32))
235fcdfe9d2SHuacai Chen goto overflow;
236fcdfe9d2SHuacai Chen
237fcdfe9d2SHuacai Chen /* (*(uint32_t *) PC) = opr */
238fcdfe9d2SHuacai Chen *location = (u32)opr1;
239fcdfe9d2SHuacai Chen return 0;
240fcdfe9d2SHuacai Chen default:
241fcdfe9d2SHuacai Chen pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
242fcdfe9d2SHuacai Chen return -EINVAL;
243fcdfe9d2SHuacai Chen }
244fcdfe9d2SHuacai Chen
245fcdfe9d2SHuacai Chen overflow:
246fcdfe9d2SHuacai Chen pr_err("module %s: opr1 = 0x%llx overflow! dangerous %s (%u) relocation\n",
247fcdfe9d2SHuacai Chen mod->name, opr1, __func__, type);
248fcdfe9d2SHuacai Chen return -ENOEXEC;
249fcdfe9d2SHuacai Chen
250fcdfe9d2SHuacai Chen unaligned:
251fcdfe9d2SHuacai Chen pr_err("module %s: opr1 = 0x%llx unaligned! dangerous %s (%u) relocation\n",
252fcdfe9d2SHuacai Chen mod->name, opr1, __func__, type);
253fcdfe9d2SHuacai Chen return -ENOEXEC;
254fcdfe9d2SHuacai Chen }
255fcdfe9d2SHuacai Chen
apply_r_larch_add_sub(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)256fcdfe9d2SHuacai Chen static int apply_r_larch_add_sub(struct module *mod, u32 *location, Elf_Addr v,
257fcdfe9d2SHuacai Chen s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
258fcdfe9d2SHuacai Chen {
259fcdfe9d2SHuacai Chen switch (type) {
260fcdfe9d2SHuacai Chen case R_LARCH_ADD32:
261fcdfe9d2SHuacai Chen *(s32 *)location += v;
262fcdfe9d2SHuacai Chen return 0;
263fcdfe9d2SHuacai Chen case R_LARCH_ADD64:
264fcdfe9d2SHuacai Chen *(s64 *)location += v;
265fcdfe9d2SHuacai Chen return 0;
266fcdfe9d2SHuacai Chen case R_LARCH_SUB32:
267fcdfe9d2SHuacai Chen *(s32 *)location -= v;
268fcdfe9d2SHuacai Chen return 0;
269fcdfe9d2SHuacai Chen case R_LARCH_SUB64:
270fcdfe9d2SHuacai Chen *(s64 *)location -= v;
271fcdfe9d2SHuacai Chen return 0;
272fcdfe9d2SHuacai Chen default:
273fcdfe9d2SHuacai Chen pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
274fcdfe9d2SHuacai Chen return -EINVAL;
275fcdfe9d2SHuacai Chen }
276fcdfe9d2SHuacai Chen }
277fcdfe9d2SHuacai Chen
apply_r_larch_b26(struct module * mod,Elf_Shdr * sechdrs,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)2789151dde4SHuacai Chen static int apply_r_larch_b26(struct module *mod,
2799151dde4SHuacai Chen Elf_Shdr *sechdrs, u32 *location, Elf_Addr v,
2809bd1e380SXi Ruoyao s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
2819bd1e380SXi Ruoyao {
2829bd1e380SXi Ruoyao ptrdiff_t offset = (void *)v - (void *)location;
2839bd1e380SXi Ruoyao union loongarch_instruction *insn = (union loongarch_instruction *)location;
2849bd1e380SXi Ruoyao
2859bd1e380SXi Ruoyao if (offset >= SZ_128M)
2869151dde4SHuacai Chen v = module_emit_plt_entry(mod, sechdrs, v);
2879bd1e380SXi Ruoyao
2889bd1e380SXi Ruoyao if (offset < -SZ_128M)
2899151dde4SHuacai Chen v = module_emit_plt_entry(mod, sechdrs, v);
2909bd1e380SXi Ruoyao
2919bd1e380SXi Ruoyao offset = (void *)v - (void *)location;
2929bd1e380SXi Ruoyao
2939bd1e380SXi Ruoyao if (offset & 3) {
2949bd1e380SXi Ruoyao pr_err("module %s: jump offset = 0x%llx unaligned! dangerous R_LARCH_B26 (%u) relocation\n",
2959bd1e380SXi Ruoyao mod->name, (long long)offset, type);
2969bd1e380SXi Ruoyao return -ENOEXEC;
2979bd1e380SXi Ruoyao }
2989bd1e380SXi Ruoyao
2999bd1e380SXi Ruoyao if (!signed_imm_check(offset, 28)) {
3009bd1e380SXi Ruoyao pr_err("module %s: jump offset = 0x%llx overflow! dangerous R_LARCH_B26 (%u) relocation\n",
3019bd1e380SXi Ruoyao mod->name, (long long)offset, type);
3029bd1e380SXi Ruoyao return -ENOEXEC;
3039bd1e380SXi Ruoyao }
3049bd1e380SXi Ruoyao
3059bd1e380SXi Ruoyao offset >>= 2;
3069bd1e380SXi Ruoyao insn->reg0i26_format.immediate_l = offset & 0xffff;
3079bd1e380SXi Ruoyao insn->reg0i26_format.immediate_h = (offset >> 16) & 0x3ff;
3089bd1e380SXi Ruoyao
3099bd1e380SXi Ruoyao return 0;
3109bd1e380SXi Ruoyao }
3119bd1e380SXi Ruoyao
apply_r_larch_pcala(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)3129bd1e380SXi Ruoyao static int apply_r_larch_pcala(struct module *mod, u32 *location, Elf_Addr v,
3139bd1e380SXi Ruoyao s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
3149bd1e380SXi Ruoyao {
3159bd1e380SXi Ruoyao union loongarch_instruction *insn = (union loongarch_instruction *)location;
3169bd1e380SXi Ruoyao /* Use s32 for a sign-extension deliberately. */
3179bd1e380SXi Ruoyao s32 offset_hi20 = (void *)((v + 0x800) & ~0xfff) -
3189bd1e380SXi Ruoyao (void *)((Elf_Addr)location & ~0xfff);
3199bd1e380SXi Ruoyao Elf_Addr anchor = (((Elf_Addr)location) & ~0xfff) + offset_hi20;
3209bd1e380SXi Ruoyao ptrdiff_t offset_rem = (void *)v - (void *)anchor;
3219bd1e380SXi Ruoyao
3229bd1e380SXi Ruoyao switch (type) {
3239bd1e380SXi Ruoyao case R_LARCH_PCALA_LO12:
3249bd1e380SXi Ruoyao insn->reg2i12_format.immediate = v & 0xfff;
3259bd1e380SXi Ruoyao break;
3269bd1e380SXi Ruoyao case R_LARCH_PCALA_HI20:
3279bd1e380SXi Ruoyao v = offset_hi20 >> 12;
3289bd1e380SXi Ruoyao insn->reg1i20_format.immediate = v & 0xfffff;
3299bd1e380SXi Ruoyao break;
3309bd1e380SXi Ruoyao case R_LARCH_PCALA64_LO20:
3319bd1e380SXi Ruoyao v = offset_rem >> 32;
3329bd1e380SXi Ruoyao insn->reg1i20_format.immediate = v & 0xfffff;
3339bd1e380SXi Ruoyao break;
3349bd1e380SXi Ruoyao case R_LARCH_PCALA64_HI12:
3359bd1e380SXi Ruoyao v = offset_rem >> 52;
3369bd1e380SXi Ruoyao insn->reg2i12_format.immediate = v & 0xfff;
3379bd1e380SXi Ruoyao break;
3389bd1e380SXi Ruoyao default:
3399bd1e380SXi Ruoyao pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
3409bd1e380SXi Ruoyao return -EINVAL;
3419bd1e380SXi Ruoyao }
3429bd1e380SXi Ruoyao
3439bd1e380SXi Ruoyao return 0;
3449bd1e380SXi Ruoyao }
3459bd1e380SXi Ruoyao
apply_r_larch_got_pc(struct module * mod,Elf_Shdr * sechdrs,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)3469151dde4SHuacai Chen static int apply_r_larch_got_pc(struct module *mod,
3479151dde4SHuacai Chen Elf_Shdr *sechdrs, u32 *location, Elf_Addr v,
34859b3d4a9SXi Ruoyao s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
34959b3d4a9SXi Ruoyao {
3509151dde4SHuacai Chen Elf_Addr got = module_emit_got_entry(mod, sechdrs, v);
35159b3d4a9SXi Ruoyao
35259b3d4a9SXi Ruoyao if (!got)
35359b3d4a9SXi Ruoyao return -EINVAL;
35459b3d4a9SXi Ruoyao
35559b3d4a9SXi Ruoyao switch (type) {
35659b3d4a9SXi Ruoyao case R_LARCH_GOT_PC_LO12:
35759b3d4a9SXi Ruoyao type = R_LARCH_PCALA_LO12;
35859b3d4a9SXi Ruoyao break;
35959b3d4a9SXi Ruoyao case R_LARCH_GOT_PC_HI20:
36059b3d4a9SXi Ruoyao type = R_LARCH_PCALA_HI20;
36159b3d4a9SXi Ruoyao break;
36259b3d4a9SXi Ruoyao default:
36359b3d4a9SXi Ruoyao pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
36459b3d4a9SXi Ruoyao return -EINVAL;
36559b3d4a9SXi Ruoyao }
36659b3d4a9SXi Ruoyao
36759b3d4a9SXi Ruoyao return apply_r_larch_pcala(mod, location, got, rela_stack, rela_stack_top, type);
36859b3d4a9SXi Ruoyao }
36959b3d4a9SXi Ruoyao
apply_r_larch_32_pcrel(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)370c1c2ce2dSTiezhu Yang static int apply_r_larch_32_pcrel(struct module *mod, u32 *location, Elf_Addr v,
371c1c2ce2dSTiezhu Yang s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
372c1c2ce2dSTiezhu Yang {
373c1c2ce2dSTiezhu Yang ptrdiff_t offset = (void *)v - (void *)location;
374c1c2ce2dSTiezhu Yang
375c1c2ce2dSTiezhu Yang *(u32 *)location = offset;
376c1c2ce2dSTiezhu Yang return 0;
377c1c2ce2dSTiezhu Yang }
378c1c2ce2dSTiezhu Yang
apply_r_larch_64_pcrel(struct module * mod,u32 * location,Elf_Addr v,s64 * rela_stack,size_t * rela_stack_top,unsigned int type)379*b1dc55a3STiezhu Yang static int apply_r_larch_64_pcrel(struct module *mod, u32 *location, Elf_Addr v,
380*b1dc55a3STiezhu Yang s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
381*b1dc55a3STiezhu Yang {
382*b1dc55a3STiezhu Yang ptrdiff_t offset = (void *)v - (void *)location;
383*b1dc55a3STiezhu Yang
384*b1dc55a3STiezhu Yang *(u64 *)location = offset;
385*b1dc55a3STiezhu Yang return 0;
386*b1dc55a3STiezhu Yang }
387*b1dc55a3STiezhu Yang
388fcdfe9d2SHuacai Chen /*
389fcdfe9d2SHuacai Chen * reloc_handlers_rela() - Apply a particular relocation to a module
390fcdfe9d2SHuacai Chen * @mod: the module to apply the reloc to
391fcdfe9d2SHuacai Chen * @location: the address at which the reloc is to be applied
392fcdfe9d2SHuacai Chen * @v: the value of the reloc, with addend for RELA-style
393fcdfe9d2SHuacai Chen * @rela_stack: the stack used for store relocation info, LOCAL to THIS module
394fcdfe9d2SHuacai Chen * @rela_stac_top: where the stack operation(pop/push) applies to
395fcdfe9d2SHuacai Chen *
396fcdfe9d2SHuacai Chen * Return: 0 upon success, else -ERRNO
397fcdfe9d2SHuacai Chen */
398fcdfe9d2SHuacai Chen typedef int (*reloc_rela_handler)(struct module *mod, u32 *location, Elf_Addr v,
399fcdfe9d2SHuacai Chen s64 *rela_stack, size_t *rela_stack_top, unsigned int type);
400fcdfe9d2SHuacai Chen
401fcdfe9d2SHuacai Chen /* The handlers for known reloc types */
402fcdfe9d2SHuacai Chen static reloc_rela_handler reloc_rela_handlers[] = {
40327614988STiezhu Yang [R_LARCH_NONE ... R_LARCH_64_PCREL] = apply_r_larch_error,
404fcdfe9d2SHuacai Chen
405fcdfe9d2SHuacai Chen [R_LARCH_NONE] = apply_r_larch_none,
406fcdfe9d2SHuacai Chen [R_LARCH_32] = apply_r_larch_32,
407fcdfe9d2SHuacai Chen [R_LARCH_64] = apply_r_larch_64,
408fcdfe9d2SHuacai Chen [R_LARCH_MARK_LA] = apply_r_larch_none,
409fcdfe9d2SHuacai Chen [R_LARCH_MARK_PCREL] = apply_r_larch_none,
410fcdfe9d2SHuacai Chen [R_LARCH_SOP_PUSH_PCREL] = apply_r_larch_sop_push_pcrel,
411fcdfe9d2SHuacai Chen [R_LARCH_SOP_PUSH_ABSOLUTE] = apply_r_larch_sop_push_absolute,
412fcdfe9d2SHuacai Chen [R_LARCH_SOP_PUSH_DUP] = apply_r_larch_sop_push_dup,
413fcdfe9d2SHuacai Chen [R_LARCH_SOP_SUB ... R_LARCH_SOP_IF_ELSE] = apply_r_larch_sop,
414fcdfe9d2SHuacai Chen [R_LARCH_SOP_POP_32_S_10_5 ... R_LARCH_SOP_POP_32_U] = apply_r_larch_sop_imm_field,
415fcdfe9d2SHuacai Chen [R_LARCH_ADD32 ... R_LARCH_SUB64] = apply_r_larch_add_sub,
4169bd1e380SXi Ruoyao [R_LARCH_PCALA_HI20...R_LARCH_PCALA64_HI12] = apply_r_larch_pcala,
417c1c2ce2dSTiezhu Yang [R_LARCH_32_PCREL] = apply_r_larch_32_pcrel,
418*b1dc55a3STiezhu Yang [R_LARCH_64_PCREL] = apply_r_larch_64_pcrel,
419fcdfe9d2SHuacai Chen };
420fcdfe9d2SHuacai Chen
apply_relocate_add(Elf_Shdr * sechdrs,const char * strtab,unsigned int symindex,unsigned int relsec,struct module * mod)421fcdfe9d2SHuacai Chen int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
422fcdfe9d2SHuacai Chen unsigned int symindex, unsigned int relsec,
423fcdfe9d2SHuacai Chen struct module *mod)
424fcdfe9d2SHuacai Chen {
425fcdfe9d2SHuacai Chen int i, err;
426fcdfe9d2SHuacai Chen unsigned int type;
427fcdfe9d2SHuacai Chen s64 rela_stack[RELA_STACK_DEPTH];
428fcdfe9d2SHuacai Chen size_t rela_stack_top = 0;
429fcdfe9d2SHuacai Chen reloc_rela_handler handler;
430fcdfe9d2SHuacai Chen void *location;
431fcdfe9d2SHuacai Chen Elf_Addr v;
432fcdfe9d2SHuacai Chen Elf_Sym *sym;
433fcdfe9d2SHuacai Chen Elf_Rela *rel = (void *) sechdrs[relsec].sh_addr;
434fcdfe9d2SHuacai Chen
435fcdfe9d2SHuacai Chen pr_debug("%s: Applying relocate section %u to %u\n", __func__, relsec,
436fcdfe9d2SHuacai Chen sechdrs[relsec].sh_info);
437fcdfe9d2SHuacai Chen
438fcdfe9d2SHuacai Chen rela_stack_top = 0;
439fcdfe9d2SHuacai Chen for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) {
440fcdfe9d2SHuacai Chen /* This is where to make the change */
441fcdfe9d2SHuacai Chen location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr + rel[i].r_offset;
442fcdfe9d2SHuacai Chen /* This is the symbol it is referring to */
443fcdfe9d2SHuacai Chen sym = (Elf_Sym *)sechdrs[symindex].sh_addr + ELF_R_SYM(rel[i].r_info);
444fcdfe9d2SHuacai Chen if (IS_ERR_VALUE(sym->st_value)) {
445fcdfe9d2SHuacai Chen /* Ignore unresolved weak symbol */
446fcdfe9d2SHuacai Chen if (ELF_ST_BIND(sym->st_info) == STB_WEAK)
447fcdfe9d2SHuacai Chen continue;
448fcdfe9d2SHuacai Chen pr_warn("%s: Unknown symbol %s\n", mod->name, strtab + sym->st_name);
449fcdfe9d2SHuacai Chen return -ENOENT;
450fcdfe9d2SHuacai Chen }
451fcdfe9d2SHuacai Chen
452fcdfe9d2SHuacai Chen type = ELF_R_TYPE(rel[i].r_info);
453fcdfe9d2SHuacai Chen
454fcdfe9d2SHuacai Chen if (type < ARRAY_SIZE(reloc_rela_handlers))
455fcdfe9d2SHuacai Chen handler = reloc_rela_handlers[type];
456fcdfe9d2SHuacai Chen else
457fcdfe9d2SHuacai Chen handler = NULL;
458fcdfe9d2SHuacai Chen
459fcdfe9d2SHuacai Chen if (!handler) {
460fcdfe9d2SHuacai Chen pr_err("%s: Unknown relocation type %u\n", mod->name, type);
461fcdfe9d2SHuacai Chen return -EINVAL;
462fcdfe9d2SHuacai Chen }
463fcdfe9d2SHuacai Chen
464fcdfe9d2SHuacai Chen pr_debug("type %d st_value %llx r_addend %llx loc %llx\n",
465fcdfe9d2SHuacai Chen (int)ELF_R_TYPE(rel[i].r_info),
466fcdfe9d2SHuacai Chen sym->st_value, rel[i].r_addend, (u64)location);
467fcdfe9d2SHuacai Chen
468fcdfe9d2SHuacai Chen v = sym->st_value + rel[i].r_addend;
4699151dde4SHuacai Chen switch (type) {
4709151dde4SHuacai Chen case R_LARCH_B26:
4719151dde4SHuacai Chen err = apply_r_larch_b26(mod, sechdrs, location,
4729151dde4SHuacai Chen v, rela_stack, &rela_stack_top, type);
4739151dde4SHuacai Chen break;
4749151dde4SHuacai Chen case R_LARCH_GOT_PC_HI20...R_LARCH_GOT_PC_LO12:
4759151dde4SHuacai Chen err = apply_r_larch_got_pc(mod, sechdrs, location,
4769151dde4SHuacai Chen v, rela_stack, &rela_stack_top, type);
4779151dde4SHuacai Chen break;
4789151dde4SHuacai Chen case R_LARCH_SOP_PUSH_PLT_PCREL:
4799151dde4SHuacai Chen err = apply_r_larch_sop_push_plt_pcrel(mod, sechdrs, location,
4809151dde4SHuacai Chen v, rela_stack, &rela_stack_top, type);
4819151dde4SHuacai Chen break;
4829151dde4SHuacai Chen default:
483fcdfe9d2SHuacai Chen err = handler(mod, location, v, rela_stack, &rela_stack_top, type);
4849151dde4SHuacai Chen }
485fcdfe9d2SHuacai Chen if (err)
486fcdfe9d2SHuacai Chen return err;
487fcdfe9d2SHuacai Chen }
488fcdfe9d2SHuacai Chen
489fcdfe9d2SHuacai Chen return 0;
490fcdfe9d2SHuacai Chen }
491fcdfe9d2SHuacai Chen
module_alloc(unsigned long size)492fcdfe9d2SHuacai Chen void *module_alloc(unsigned long size)
493fcdfe9d2SHuacai Chen {
494fcdfe9d2SHuacai Chen return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END,
495fcdfe9d2SHuacai Chen GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0));
496fcdfe9d2SHuacai Chen }
49719e5eb15SHuacai Chen
module_init_ftrace_plt(const Elf_Ehdr * hdr,const Elf_Shdr * sechdrs,struct module * mod)49828ac0a9eSQing Zhang static void module_init_ftrace_plt(const Elf_Ehdr *hdr,
49928ac0a9eSQing Zhang const Elf_Shdr *sechdrs, struct module *mod)
50028ac0a9eSQing Zhang {
50128ac0a9eSQing Zhang #ifdef CONFIG_DYNAMIC_FTRACE
50228ac0a9eSQing Zhang struct plt_entry *ftrace_plts;
50328ac0a9eSQing Zhang
50428ac0a9eSQing Zhang ftrace_plts = (void *)sechdrs->sh_addr;
50528ac0a9eSQing Zhang
50628ac0a9eSQing Zhang ftrace_plts[FTRACE_PLT_IDX] = emit_plt_entry(FTRACE_ADDR);
50728ac0a9eSQing Zhang
50828ac0a9eSQing Zhang if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS))
50928ac0a9eSQing Zhang ftrace_plts[FTRACE_REGS_PLT_IDX] = emit_plt_entry(FTRACE_REGS_ADDR);
51028ac0a9eSQing Zhang
51128ac0a9eSQing Zhang mod->arch.ftrace_trampolines = ftrace_plts;
51228ac0a9eSQing Zhang #endif
51328ac0a9eSQing Zhang }
51428ac0a9eSQing Zhang
module_finalize(const Elf_Ehdr * hdr,const Elf_Shdr * sechdrs,struct module * mod)51519e5eb15SHuacai Chen int module_finalize(const Elf_Ehdr *hdr,
51619e5eb15SHuacai Chen const Elf_Shdr *sechdrs, struct module *mod)
51719e5eb15SHuacai Chen {
51819e5eb15SHuacai Chen const Elf_Shdr *s, *se;
51919e5eb15SHuacai Chen const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
52019e5eb15SHuacai Chen
52119e5eb15SHuacai Chen for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) {
52219e5eb15SHuacai Chen if (!strcmp(".altinstructions", secstrs + s->sh_name))
52319e5eb15SHuacai Chen apply_alternatives((void *)s->sh_addr, (void *)s->sh_addr + s->sh_size);
52428ac0a9eSQing Zhang if (!strcmp(".ftrace_trampoline", secstrs + s->sh_name))
52528ac0a9eSQing Zhang module_init_ftrace_plt(hdr, s, mod);
52619e5eb15SHuacai Chen }
52719e5eb15SHuacai Chen
52819e5eb15SHuacai Chen return 0;
52919e5eb15SHuacai Chen }
530