Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
module: replace module_layout with module_memory
module_layout manages different types of memory (text, data, rodata, etc.)
in one allocation, which is problematic for some reasons:
1. It is hard to enable CONFIG_STRICT_MODULE_RWX.
2. It is hard to use huge pages in modules (and not break strict rwx).
3. Many archs uses module_layout for arch-specific data, but it is not
obvious how these data are used (are they RO, RX, or RW?)
Improve the scenario by replacing 2 (or 3) module_layout per module with
up to 7 module_memory per module:
MOD_TEXT,
MOD_DATA,
MOD_RODATA,
MOD_RO_AFTER_INIT,
MOD_INIT_TEXT,
MOD_INIT_DATA,
MOD_INIT_RODATA,
and allocating them separately. This adds slightly more entries to
mod_tree (from up to 3 entries per module, to up to 7 entries per
module). However, this at most adds a small constant overhead to
__module_address(), which is expected to be fast.
Various archs use module_layout for different data. These data are put
into different module_memory based on their location in module_layout.
IOW, data that used to go with text is allocated with MOD_MEM_TYPE_TEXT;
data that used to go with data is allocated with MOD_MEM_TYPE_DATA, etc.
module_memory simplifies quite some of the module code. For example,
ARCH_WANTS_MODULES_DATA_IN_VMALLOC is a lot cleaner, as it just uses a
different allocator for the data. kernel/module/strict_rwx.c is also
much cleaner with module_memory.
Signed-off-by: Song Liu <song@kernel.org>
Cc: Luis Chamberlain <mcgrof@kernel.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Guenter Roeck <linux@roeck-us.net>
Cc: Christophe Leroy <christophe.leroy@csgroup.eu>
Reviewed-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Christophe Leroy <christophe.leroy@csgroup.eu>
Reviewed-by: Luis Chamberlain <mcgrof@kernel.org>
Signed-off-by: Luis Chamberlain <mcgrof@kernel.org>
authored by
Song Liu
and committed by
Luis Chamberlain
ac3b4328
fe15c26e
+437
-477
+7
-5
arch/arc/kernel/unwind.c
+7
-5
arch/arc/kernel/unwind.c
···
369
369
unsigned long table_size)
370
370
{
371
371
struct unwind_table *table;
372
+
struct module_memory *core_text;
373
+
struct module_memory *init_text;
372
374
373
375
if (table_size <= 0)
374
376
return NULL;
···
379
377
if (!table)
380
378
return NULL;
381
379
382
-
init_unwind_table(table, module->name,
383
-
module->core_layout.base, module->core_layout.size,
384
-
module->init_layout.base, module->init_layout.size,
385
-
table_start, table_size,
386
-
NULL, 0);
380
+
core_text = &module->mem[MOD_TEXT];
381
+
init_text = &module->mem[MOD_INIT_TEXT];
382
+
383
+
init_unwind_table(table, module->name, core_text->base, core_text->size,
384
+
init_text->base, init_text->size, table_start, table_size, NULL, 0);
387
385
388
386
init_unwind_hdr(table, unw_hdr_alloc);
389
387
+2
-7
arch/arm/kernel/module-plts.c
+2
-7
arch/arm/kernel/module-plts.c
···
28
28
#endif
29
29
};
30
30
31
-
static bool in_init(const struct module *mod, unsigned long loc)
32
-
{
33
-
return loc - (u32)mod->init_layout.base < mod->init_layout.size;
34
-
}
35
-
36
31
static void prealloc_fixed(struct mod_plt_sec *pltsec, struct plt_entries *plt)
37
32
{
38
33
int i;
···
45
50
46
51
u32 get_module_plt(struct module *mod, unsigned long loc, Elf32_Addr val)
47
52
{
48
-
struct mod_plt_sec *pltsec = !in_init(mod, loc) ? &mod->arch.core :
49
-
&mod->arch.init;
53
+
struct mod_plt_sec *pltsec = !within_module_init(loc, mod) ?
54
+
&mod->arch.core : &mod->arch.init;
50
55
struct plt_entries *plt;
51
56
int idx;
52
57
+4
-9
arch/arm64/kernel/module-plts.c
+4
-9
arch/arm64/kernel/module-plts.c
···
65
65
(q + aarch64_insn_adrp_get_offset(le32_to_cpu(b->adrp)));
66
66
}
67
67
68
-
static bool in_init(const struct module *mod, void *loc)
69
-
{
70
-
return (u64)loc - (u64)mod->init_layout.base < mod->init_layout.size;
71
-
}
72
-
73
68
u64 module_emit_plt_entry(struct module *mod, Elf64_Shdr *sechdrs,
74
69
void *loc, const Elf64_Rela *rela,
75
70
Elf64_Sym *sym)
76
71
{
77
-
struct mod_plt_sec *pltsec = !in_init(mod, loc) ? &mod->arch.core :
78
-
&mod->arch.init;
72
+
struct mod_plt_sec *pltsec = !within_module_init((unsigned long)loc, mod) ?
73
+
&mod->arch.core : &mod->arch.init;
79
74
struct plt_entry *plt = (struct plt_entry *)sechdrs[pltsec->plt_shndx].sh_addr;
80
75
int i = pltsec->plt_num_entries;
81
76
int j = i - 1;
···
100
105
u64 module_emit_veneer_for_adrp(struct module *mod, Elf64_Shdr *sechdrs,
101
106
void *loc, u64 val)
102
107
{
103
-
struct mod_plt_sec *pltsec = !in_init(mod, loc) ? &mod->arch.core :
104
-
&mod->arch.init;
108
+
struct mod_plt_sec *pltsec = !within_module_init((unsigned long)loc, mod) ?
109
+
&mod->arch.core : &mod->arch.init;
105
110
struct plt_entry *plt = (struct plt_entry *)sechdrs[pltsec->plt_shndx].sh_addr;
106
111
int i = pltsec->plt_num_entries++;
107
112
u32 br;
+14
-10
arch/ia64/kernel/module.c
+14
-10
arch/ia64/kernel/module.c
···
485
485
return 0;
486
486
}
487
487
488
-
static inline int
488
+
static inline bool
489
489
in_init (const struct module *mod, uint64_t addr)
490
490
{
491
-
return addr - (uint64_t) mod->init_layout.base < mod->init_layout.size;
491
+
return within_module_init(addr, mod);
492
492
}
493
493
494
-
static inline int
494
+
static inline bool
495
495
in_core (const struct module *mod, uint64_t addr)
496
496
{
497
-
return addr - (uint64_t) mod->core_layout.base < mod->core_layout.size;
497
+
return within_module_core(addr, mod);
498
498
}
499
499
500
-
static inline int
500
+
static inline bool
501
501
is_internal (const struct module *mod, uint64_t value)
502
502
{
503
503
return in_init(mod, value) || in_core(mod, value);
···
677
677
break;
678
678
679
679
case RV_BDREL:
680
-
val -= (uint64_t) (in_init(mod, val) ? mod->init_layout.base : mod->core_layout.base);
680
+
val -= (uint64_t) (in_init(mod, val) ? mod->mem[MOD_INIT_TEXT].base :
681
+
mod->mem[MOD_TEXT].base);
681
682
break;
682
683
683
684
case RV_LTV:
···
813
812
* addresses have been selected...
814
813
*/
815
814
uint64_t gp;
816
-
if (mod->core_layout.size > MAX_LTOFF)
815
+
struct module_memory *mod_mem;
816
+
817
+
mod_mem = &mod->mem[MOD_DATA];
818
+
if (mod_mem->size > MAX_LTOFF)
817
819
/*
818
820
* This takes advantage of fact that SHF_ARCH_SMALL gets allocated
819
821
* at the end of the module.
820
822
*/
821
-
gp = mod->core_layout.size - MAX_LTOFF / 2;
823
+
gp = mod_mem->size - MAX_LTOFF / 2;
822
824
else
823
-
gp = mod->core_layout.size / 2;
824
-
gp = (uint64_t) mod->core_layout.base + ((gp + 7) & -8);
825
+
gp = mod_mem->size / 2;
826
+
gp = (uint64_t) mod_mem->base + ((gp + 7) & -8);
825
827
mod->arch.gp = gp;
826
828
DEBUGP("%s: placing gp at 0x%lx\n", __func__, gp);
827
829
}
+5
-6
arch/mips/kernel/vpe.c
+5
-6
arch/mips/kernel/vpe.c
···
199
199
for (m = 0; m < ARRAY_SIZE(masks); ++m) {
200
200
for (i = 0; i < hdr->e_shnum; ++i) {
201
201
Elf_Shdr *s = &sechdrs[i];
202
+
struct module_memory *mod_mem;
203
+
204
+
mod_mem = &mod->mem[MOD_TEXT];
202
205
203
206
if ((s->sh_flags & masks[m][0]) != masks[m][0]
204
207
|| (s->sh_flags & masks[m][1])
205
208
|| s->sh_entsize != ~0UL)
206
209
continue;
207
210
s->sh_entsize =
208
-
get_offset((unsigned long *)&mod->core_layout.size, s);
211
+
get_offset((unsigned long *)&mod_mem->size, s);
209
212
}
210
-
211
-
if (m == 0)
212
-
mod->core_layout.text_size = mod->core_layout.size;
213
-
214
213
}
215
214
}
216
215
···
640
641
layout_sections(&mod, hdr, sechdrs, secstrings);
641
642
}
642
643
643
-
v->load_addr = alloc_progmem(mod.core_layout.size);
644
+
v->load_addr = alloc_progmem(mod.mod_mem[MOD_TEXT].size);
644
645
if (!v->load_addr)
645
646
return -ENOMEM;
646
647
+17
-34
arch/parisc/kernel/module.c
+17
-34
arch/parisc/kernel/module.c
···
27
27
* We are not doing SEGREL32 handling correctly. According to the ABI, we
28
28
* should do a value offset, like this:
29
29
* if (in_init(me, (void *)val))
30
-
* val -= (uint32_t)me->init_layout.base;
30
+
* val -= (uint32_t)me->mem[MOD_INIT_TEXT].base;
31
31
* else
32
-
* val -= (uint32_t)me->core_layout.base;
32
+
* val -= (uint32_t)me->mem[MOD_TEXT].base;
33
33
* However, SEGREL32 is used only for PARISC unwind entries, and we want
34
34
* those entries to have an absolute address, and not just an offset.
35
35
*
···
75
75
* which gives us a maximum positive displacement of 0x7fff, and as such
76
76
* allows us to allocate up to 4095 GOT entries. */
77
77
#define MAX_GOTS 4095
78
-
79
-
/* three functions to determine where in the module core
80
-
* or init pieces the location is */
81
-
static inline int in_init(struct module *me, void *loc)
82
-
{
83
-
return (loc >= me->init_layout.base &&
84
-
loc <= (me->init_layout.base + me->init_layout.size));
85
-
}
86
-
87
-
static inline int in_core(struct module *me, void *loc)
88
-
{
89
-
return (loc >= me->core_layout.base &&
90
-
loc <= (me->core_layout.base + me->core_layout.size));
91
-
}
92
-
93
-
static inline int in_local(struct module *me, void *loc)
94
-
{
95
-
return in_init(me, loc) || in_core(me, loc);
96
-
}
97
78
98
79
#ifndef CONFIG_64BIT
99
80
struct got_entry {
···
283
302
{
284
303
unsigned long gots = 0, fdescs = 0, len;
285
304
unsigned int i;
305
+
struct module_memory *mod_mem;
286
306
287
307
len = hdr->e_shnum * sizeof(me->arch.section[0]);
288
308
me->arch.section = kzalloc(len, GFP_KERNEL);
···
328
346
me->arch.section[s].stub_entries += count;
329
347
}
330
348
349
+
mod_mem = &me->mem[MOD_TEXT];
331
350
/* align things a bit */
332
-
me->core_layout.size = ALIGN(me->core_layout.size, 16);
333
-
me->arch.got_offset = me->core_layout.size;
334
-
me->core_layout.size += gots * sizeof(struct got_entry);
351
+
mod_mem->size = ALIGN(mod_mem->size, 16);
352
+
me->arch.got_offset = mod_mem->size;
353
+
mod_mem->size += gots * sizeof(struct got_entry);
335
354
336
-
me->core_layout.size = ALIGN(me->core_layout.size, 16);
337
-
me->arch.fdesc_offset = me->core_layout.size;
338
-
me->core_layout.size += fdescs * sizeof(Elf_Fdesc);
355
+
mod_mem->size = ALIGN(mod_mem->size, 16);
356
+
me->arch.fdesc_offset = mod_mem->size;
357
+
mod_mem->size += fdescs * sizeof(Elf_Fdesc);
339
358
340
359
me->arch.got_max = gots;
341
360
me->arch.fdesc_max = fdescs;
···
354
371
355
372
BUG_ON(value == 0);
356
373
357
-
got = me->core_layout.base + me->arch.got_offset;
374
+
got = me->mem[MOD_TEXT].base + me->arch.got_offset;
358
375
for (i = 0; got[i].addr; i++)
359
376
if (got[i].addr == value)
360
377
goto out;
···
372
389
#ifdef CONFIG_64BIT
373
390
static Elf_Addr get_fdesc(struct module *me, unsigned long value)
374
391
{
375
-
Elf_Fdesc *fdesc = me->core_layout.base + me->arch.fdesc_offset;
392
+
Elf_Fdesc *fdesc = me->mem[MOD_TEXT].base + me->arch.fdesc_offset;
376
393
377
394
if (!value) {
378
395
printk(KERN_ERR "%s: zero OPD requested!\n", me->name);
···
390
407
391
408
/* Create new one */
392
409
fdesc->addr = value;
393
-
fdesc->gp = (Elf_Addr)me->core_layout.base + me->arch.got_offset;
410
+
fdesc->gp = (Elf_Addr)me->mem[MOD_TEXT].base + me->arch.got_offset;
394
411
return (Elf_Addr)fdesc;
395
412
}
396
413
#endif /* CONFIG_64BIT */
···
725
742
loc, val);
726
743
val += addend;
727
744
/* can we reach it locally? */
728
-
if (in_local(me, (void *)val)) {
745
+
if (within_module(val, me)) {
729
746
/* this is the case where the symbol is local
730
747
* to the module, but in a different section,
731
748
* so stub the jump in case it's more than 22
···
784
801
break;
785
802
case R_PARISC_FPTR64:
786
803
/* 64-bit function address */
787
-
if(in_local(me, (void *)(val + addend))) {
804
+
if (within_module(val + addend, me)) {
788
805
*loc64 = get_fdesc(me, val+addend);
789
806
pr_debug("FDESC for %s at %llx points to %llx\n",
790
807
strtab + sym->st_name, *loc64,
···
822
839
823
840
table = (unsigned char *)sechdrs[me->arch.unwind_section].sh_addr;
824
841
end = table + sechdrs[me->arch.unwind_section].sh_size;
825
-
gp = (Elf_Addr)me->core_layout.base + me->arch.got_offset;
842
+
gp = (Elf_Addr)me->mem[MOD_TEXT].base + me->arch.got_offset;
826
843
827
844
pr_debug("register_unwind_table(), sect = %d at 0x%p - 0x%p (gp=0x%lx)\n",
828
845
me->arch.unwind_section, table, end, gp);
···
960
977
#ifdef CONFIG_64BIT
961
978
void *dereference_module_function_descriptor(struct module *mod, void *ptr)
962
979
{
963
-
unsigned long start_opd = (Elf64_Addr)mod->core_layout.base +
980
+
unsigned long start_opd = (Elf64_Addr)mod->mem[MOD_TEXT].base +
964
981
mod->arch.fdesc_offset;
965
982
unsigned long end_opd = start_opd +
966
983
mod->arch.fdesc_count * sizeof(Elf64_Fdesc);
+3
-4
arch/powerpc/kernel/module_32.c
+3
-4
arch/powerpc/kernel/module_32.c
···
163
163
164
164
pr_debug("Doing plt for call to 0x%x at 0x%x\n", val, (unsigned int)location);
165
165
/* Init, or core PLT? */
166
-
if (location >= mod->core_layout.base
167
-
&& location < mod->core_layout.base + mod->core_layout.size)
166
+
if (within_module_core((unsigned long)location, mod))
168
167
entry = (void *)sechdrs[mod->arch.core_plt_section].sh_addr;
169
168
else
170
169
entry = (void *)sechdrs[mod->arch.init_plt_section].sh_addr;
···
321
322
322
323
int module_finalize_ftrace(struct module *module, const Elf_Shdr *sechdrs)
323
324
{
324
-
module->arch.tramp = do_plt_call(module->core_layout.base,
325
+
module->arch.tramp = do_plt_call(module->mem[MOD_TEXT].base,
325
326
(unsigned long)ftrace_caller,
326
327
sechdrs, module);
327
328
if (!module->arch.tramp)
328
329
return -ENOENT;
329
330
330
331
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
331
-
module->arch.tramp_regs = do_plt_call(module->core_layout.base,
332
+
module->arch.tramp_regs = do_plt_call(module->mem[MOD_TEXT].base,
332
333
(unsigned long)ftrace_regs_caller,
333
334
sechdrs, module);
334
335
if (!module->arch.tramp_regs)
+14
-12
arch/s390/kernel/module.c
+14
-12
arch/s390/kernel/module.c
···
126
126
Elf_Rela *rela;
127
127
char *strings;
128
128
int nrela, i, j;
129
+
struct module_memory *mod_mem;
129
130
130
131
/* Find symbol table and string table. */
131
132
symtab = NULL;
···
174
173
175
174
/* Increase core size by size of got & plt and set start
176
175
offsets for got and plt. */
177
-
me->core_layout.size = ALIGN(me->core_layout.size, 4);
178
-
me->arch.got_offset = me->core_layout.size;
179
-
me->core_layout.size += me->arch.got_size;
180
-
me->arch.plt_offset = me->core_layout.size;
176
+
mod_mem = &me->mem[MOD_TEXT];
177
+
mod_mem->size = ALIGN(mod_mem->size, 4);
178
+
me->arch.got_offset = mod_mem->size;
179
+
mod_mem->size += me->arch.got_size;
180
+
me->arch.plt_offset = mod_mem->size;
181
181
if (me->arch.plt_size) {
182
182
if (IS_ENABLED(CONFIG_EXPOLINE) && !nospec_disable)
183
183
me->arch.plt_size += PLT_ENTRY_SIZE;
184
-
me->core_layout.size += me->arch.plt_size;
184
+
mod_mem->size += me->arch.plt_size;
185
185
}
186
186
return 0;
187
187
}
···
306
304
case R_390_GOTPLT64: /* 64 bit offset to jump slot. */
307
305
case R_390_GOTPLTENT: /* 32 bit rel. offset to jump slot >> 1. */
308
306
if (info->got_initialized == 0) {
309
-
Elf_Addr *gotent = me->core_layout.base +
307
+
Elf_Addr *gotent = me->mem[MOD_TEXT].base +
310
308
me->arch.got_offset +
311
309
info->got_offset;
312
310
···
331
329
rc = apply_rela_bits(loc, val, 0, 64, 0, write);
332
330
else if (r_type == R_390_GOTENT ||
333
331
r_type == R_390_GOTPLTENT) {
334
-
val += (Elf_Addr) me->core_layout.base - loc;
332
+
val += (Elf_Addr) me->mem[MOD_TEXT].base - loc;
335
333
rc = apply_rela_bits(loc, val, 1, 32, 1, write);
336
334
}
337
335
break;
···
347
345
char *plt_base;
348
346
char *ip;
349
347
350
-
plt_base = me->core_layout.base + me->arch.plt_offset;
348
+
plt_base = me->mem[MOD_TEXT].base + me->arch.plt_offset;
351
349
ip = plt_base + info->plt_offset;
352
350
*(int *)insn = 0x0d10e310; /* basr 1,0 */
353
351
*(int *)&insn[4] = 0x100c0004; /* lg 1,12(1) */
···
377
375
val - loc + 0xffffUL < 0x1ffffeUL) ||
378
376
(r_type == R_390_PLT32DBL &&
379
377
val - loc + 0xffffffffULL < 0x1fffffffeULL)))
380
-
val = (Elf_Addr) me->core_layout.base +
378
+
val = (Elf_Addr) me->mem[MOD_TEXT].base +
381
379
me->arch.plt_offset +
382
380
info->plt_offset;
383
381
val += rela->r_addend - loc;
···
399
397
case R_390_GOTOFF32: /* 32 bit offset to GOT. */
400
398
case R_390_GOTOFF64: /* 64 bit offset to GOT. */
401
399
val = val + rela->r_addend -
402
-
((Elf_Addr) me->core_layout.base + me->arch.got_offset);
400
+
((Elf_Addr) me->mem[MOD_TEXT].base + me->arch.got_offset);
403
401
if (r_type == R_390_GOTOFF16)
404
402
rc = apply_rela_bits(loc, val, 0, 16, 0, write);
405
403
else if (r_type == R_390_GOTOFF32)
···
409
407
break;
410
408
case R_390_GOTPC: /* 32 bit PC relative offset to GOT. */
411
409
case R_390_GOTPCDBL: /* 32 bit PC rel. off. to GOT shifted by 1. */
412
-
val = (Elf_Addr) me->core_layout.base + me->arch.got_offset +
410
+
val = (Elf_Addr) me->mem[MOD_TEXT].base + me->arch.got_offset +
413
411
rela->r_addend - loc;
414
412
if (r_type == R_390_GOTPC)
415
413
rc = apply_rela_bits(loc, val, 1, 32, 0, write);
···
517
515
!nospec_disable && me->arch.plt_size) {
518
516
unsigned int *ij;
519
517
520
-
ij = me->core_layout.base + me->arch.plt_offset +
518
+
ij = me->mem[MOD_TEXT].base + me->arch.plt_offset +
521
519
me->arch.plt_size - PLT_ENTRY_SIZE;
522
520
ij[0] = 0xc6000000; /* exrl %r0,.+10 */
523
521
ij[1] = 0x0005a7f4; /* j . */
+2
-2
arch/x86/kernel/callthunks.c
+2
-2
arch/x86/kernel/callthunks.c
···
330
330
struct module *mod)
331
331
{
332
332
struct core_text ct = {
333
-
.base = (unsigned long)mod->core_layout.base,
334
-
.end = (unsigned long)mod->core_layout.base + mod->core_layout.size,
333
+
.base = (unsigned long)mod->mem[MOD_TEXT].base,
334
+
.end = (unsigned long)mod->mem[MOD_TEXT].base + mod->mem[MOD_TEXT].size,
335
335
.name = mod->name,
336
336
};
337
337
+2
-2
arch/x86/kernel/module.c
+2
-2
arch/x86/kernel/module.c
···
362
362
}
363
363
if (locks) {
364
364
void *lseg = (void *)locks->sh_addr;
365
-
void *text = me->core_layout.base;
366
-
void *text_end = text + me->core_layout.text_size;
365
+
void *text = me->mem[MOD_TEXT].base;
366
+
void *text_end = text + me->mem[MOD_TEXT].size;
367
367
alternatives_smp_module_add(me, me->name,
368
368
lseg, lseg + locks->sh_size,
369
369
text, text_end);
+63
-26
include/linux/module.h
+63
-26
include/linux/module.h
···
320
320
struct latch_tree_node node;
321
321
};
322
322
323
-
struct module_layout {
324
-
/* The actual code + data. */
323
+
enum mod_mem_type {
324
+
MOD_TEXT = 0,
325
+
MOD_DATA,
326
+
MOD_RODATA,
327
+
MOD_RO_AFTER_INIT,
328
+
MOD_INIT_TEXT,
329
+
MOD_INIT_DATA,
330
+
MOD_INIT_RODATA,
331
+
332
+
MOD_MEM_NUM_TYPES,
333
+
MOD_INVALID = -1,
334
+
};
335
+
336
+
#define mod_mem_type_is_init(type) \
337
+
((type) == MOD_INIT_TEXT || \
338
+
(type) == MOD_INIT_DATA || \
339
+
(type) == MOD_INIT_RODATA)
340
+
341
+
#define mod_mem_type_is_core(type) (!mod_mem_type_is_init(type))
342
+
343
+
#define mod_mem_type_is_text(type) \
344
+
((type) == MOD_TEXT || \
345
+
(type) == MOD_INIT_TEXT)
346
+
347
+
#define mod_mem_type_is_data(type) (!mod_mem_type_is_text(type))
348
+
349
+
#define mod_mem_type_is_core_data(type) \
350
+
(mod_mem_type_is_core(type) && \
351
+
mod_mem_type_is_data(type))
352
+
353
+
#define for_each_mod_mem_type(type) \
354
+
for (enum mod_mem_type (type) = 0; \
355
+
(type) < MOD_MEM_NUM_TYPES; (type)++)
356
+
357
+
#define for_class_mod_mem_type(type, class) \
358
+
for_each_mod_mem_type(type) \
359
+
if (mod_mem_type_is_##class(type))
360
+
361
+
struct module_memory {
325
362
void *base;
326
-
/* Total size. */
327
363
unsigned int size;
328
-
/* The size of the executable code. */
329
-
unsigned int text_size;
330
-
/* Size of RO section of the module (text+rodata) */
331
-
unsigned int ro_size;
332
-
/* Size of RO after init section */
333
-
unsigned int ro_after_init_size;
334
364
335
365
#ifdef CONFIG_MODULES_TREE_LOOKUP
336
366
struct mod_tree_node mtn;
···
369
339
370
340
#ifdef CONFIG_MODULES_TREE_LOOKUP
371
341
/* Only touch one cacheline for common rbtree-for-core-layout case. */
372
-
#define __module_layout_align ____cacheline_aligned
342
+
#define __module_memory_align ____cacheline_aligned
373
343
#else
374
-
#define __module_layout_align
344
+
#define __module_memory_align
375
345
#endif
376
346
377
347
struct mod_kallsyms {
···
456
426
/* Startup function. */
457
427
int (*init)(void);
458
428
459
-
/* Core layout: rbtree is accessed frequently, so keep together. */
460
-
struct module_layout core_layout __module_layout_align;
461
-
struct module_layout init_layout;
462
-
#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
463
-
struct module_layout data_layout;
464
-
#endif
429
+
struct module_memory mem[MOD_MEM_NUM_TYPES] __module_memory_align;
465
430
466
431
/* Arch-specific module values */
467
432
struct mod_arch_specific arch;
···
606
581
bool is_module_percpu_address(unsigned long addr);
607
582
bool is_module_text_address(unsigned long addr);
608
583
584
+
static inline bool within_module_mem_type(unsigned long addr,
585
+
const struct module *mod,
586
+
enum mod_mem_type type)
587
+
{
588
+
unsigned long base, size;
589
+
590
+
base = (unsigned long)mod->mem[type].base;
591
+
size = mod->mem[type].size;
592
+
return addr - base < size;
593
+
}
594
+
609
595
static inline bool within_module_core(unsigned long addr,
610
596
const struct module *mod)
611
597
{
612
-
#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
613
-
if ((unsigned long)mod->data_layout.base <= addr &&
614
-
addr < (unsigned long)mod->data_layout.base + mod->data_layout.size)
615
-
return true;
616
-
#endif
617
-
return (unsigned long)mod->core_layout.base <= addr &&
618
-
addr < (unsigned long)mod->core_layout.base + mod->core_layout.size;
598
+
for_class_mod_mem_type(type, core) {
599
+
if (within_module_mem_type(addr, mod, type))
600
+
return true;
601
+
}
602
+
return false;
619
603
}
620
604
621
605
static inline bool within_module_init(unsigned long addr,
622
606
const struct module *mod)
623
607
{
624
-
return (unsigned long)mod->init_layout.base <= addr &&
625
-
addr < (unsigned long)mod->init_layout.base + mod->init_layout.size;
608
+
for_class_mod_mem_type(type, init) {
609
+
if (within_module_mem_type(addr, mod, type))
610
+
return true;
611
+
}
612
+
return false;
626
613
}
627
614
628
615
static inline bool within_module(unsigned long addr, const struct module *mod)
+17
-23
kernel/module/internal.h
+17
-23
kernel/module/internal.h
···
17
17
#define ARCH_SHF_SMALL 0
18
18
#endif
19
19
20
-
/* If this is set, the section belongs in the init part of the module */
21
-
#define INIT_OFFSET_MASK (1UL << (BITS_PER_LONG - 1))
20
+
/*
21
+
* Use highest 4 bits of sh_entsize to store the mod_mem_type of this
22
+
* section. This leaves 28 bits for offset on 32-bit systems, which is
23
+
* about 256 MiB (WARN_ON_ONCE if we exceed that).
24
+
*/
25
+
26
+
#define SH_ENTSIZE_TYPE_BITS 4
27
+
#define SH_ENTSIZE_TYPE_SHIFT (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS)
28
+
#define SH_ENTSIZE_TYPE_MASK ((1UL << SH_ENTSIZE_TYPE_BITS) - 1)
29
+
#define SH_ENTSIZE_OFFSET_MASK ((1UL << (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS)) - 1)
30
+
22
31
/* Maximum number of characters written by module_flags() */
23
32
#define MODULE_FLAGS_BUF_SIZE (TAINT_FLAGS_COUNT + 4)
24
-
25
-
#ifndef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
26
-
#define data_layout core_layout
27
-
#endif
28
-
29
-
/*
30
-
* Modules' sections will be aligned on page boundaries
31
-
* to ensure complete separation of code and data, but
32
-
* only when CONFIG_STRICT_MODULE_RWX=y
33
-
*/
34
-
static inline unsigned int strict_align(unsigned int size)
35
-
{
36
-
if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
37
-
return PAGE_ALIGN(size);
38
-
else
39
-
return size;
40
-
}
41
33
42
34
extern struct mutex module_mutex;
43
35
extern struct list_head modules;
···
93
101
bool find_symbol(struct find_symbol_arg *fsa);
94
102
struct module *find_module_all(const char *name, size_t len, bool even_unformed);
95
103
int cmp_name(const void *name, const void *sym);
96
-
long module_get_offset(struct module *mod, unsigned int *size, Elf_Shdr *sechdr,
97
-
unsigned int section);
104
+
long module_get_offset_and_type(struct module *mod, enum mod_mem_type type,
105
+
Elf_Shdr *sechdr, unsigned int section);
98
106
char *module_flags(struct module *mod, char *buf, bool show_state);
99
107
size_t module_flags_taint(unsigned long taints, char *buf);
100
108
···
182
190
#endif
183
191
unsigned long addr_min;
184
192
unsigned long addr_max;
193
+
#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
194
+
unsigned long data_addr_min;
195
+
unsigned long data_addr_max;
196
+
#endif
185
197
};
186
198
187
199
extern struct mod_tree_root mod_tree;
188
-
extern struct mod_tree_root mod_data_tree;
189
200
190
201
#ifdef CONFIG_MODULES_TREE_LOOKUP
191
202
void mod_tree_insert(struct module *mod);
···
219
224
void module_enable_x(const struct module *mod);
220
225
int module_enforce_rwx_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs,
221
226
char *secstrings, struct module *mod);
222
-
bool module_check_misalignment(const struct module *mod);
223
227
224
228
#ifdef CONFIG_MODULE_SIG
225
229
int module_sig_check(struct load_info *info, int flags);
+33
-25
kernel/module/kallsyms.c
+33
-25
kernel/module/kallsyms.c
···
78
78
unsigned int shnum, unsigned int pcpundx)
79
79
{
80
80
const Elf_Shdr *sec;
81
+
enum mod_mem_type type;
81
82
82
83
if (src->st_shndx == SHN_UNDEF ||
83
84
src->st_shndx >= shnum ||
···
91
90
#endif
92
91
93
92
sec = sechdrs + src->st_shndx;
93
+
type = sec->sh_entsize >> SH_ENTSIZE_TYPE_SHIFT;
94
94
if (!(sec->sh_flags & SHF_ALLOC)
95
95
#ifndef CONFIG_KALLSYMS_ALL
96
96
|| !(sec->sh_flags & SHF_EXECINSTR)
97
97
#endif
98
-
|| (sec->sh_entsize & INIT_OFFSET_MASK))
98
+
|| mod_mem_type_is_init(type))
99
99
return false;
100
100
101
101
return true;
···
115
113
Elf_Shdr *strsect = info->sechdrs + info->index.str;
116
114
const Elf_Sym *src;
117
115
unsigned int i, nsrc, ndst, strtab_size = 0;
116
+
struct module_memory *mod_mem_data = &mod->mem[MOD_DATA];
117
+
struct module_memory *mod_mem_init_data = &mod->mem[MOD_INIT_DATA];
118
118
119
119
/* Put symbol section at end of init part of module. */
120
120
symsect->sh_flags |= SHF_ALLOC;
121
-
symsect->sh_entsize = module_get_offset(mod, &mod->init_layout.size, symsect,
122
-
info->index.sym) | INIT_OFFSET_MASK;
121
+
symsect->sh_entsize = module_get_offset_and_type(mod, MOD_INIT_DATA,
122
+
symsect, info->index.sym);
123
123
pr_debug("\t%s\n", info->secstrings + symsect->sh_name);
124
124
125
125
src = (void *)info->hdr + symsect->sh_offset;
···
138
134
}
139
135
140
136
/* Append room for core symbols at end of core part. */
141
-
info->symoffs = ALIGN(mod->data_layout.size, symsect->sh_addralign ?: 1);
142
-
info->stroffs = mod->data_layout.size = info->symoffs + ndst * sizeof(Elf_Sym);
143
-
mod->data_layout.size += strtab_size;
137
+
info->symoffs = ALIGN(mod_mem_data->size, symsect->sh_addralign ?: 1);
138
+
info->stroffs = mod_mem_data->size = info->symoffs + ndst * sizeof(Elf_Sym);
139
+
mod_mem_data->size += strtab_size;
144
140
/* Note add_kallsyms() computes strtab_size as core_typeoffs - stroffs */
145
-
info->core_typeoffs = mod->data_layout.size;
146
-
mod->data_layout.size += ndst * sizeof(char);
147
-
mod->data_layout.size = strict_align(mod->data_layout.size);
141
+
info->core_typeoffs = mod_mem_data->size;
142
+
mod_mem_data->size += ndst * sizeof(char);
148
143
149
144
/* Put string table section at end of init part of module. */
150
145
strsect->sh_flags |= SHF_ALLOC;
151
-
strsect->sh_entsize = module_get_offset(mod, &mod->init_layout.size, strsect,
152
-
info->index.str) | INIT_OFFSET_MASK;
146
+
strsect->sh_entsize = module_get_offset_and_type(mod, MOD_INIT_DATA,
147
+
strsect, info->index.str);
153
148
pr_debug("\t%s\n", info->secstrings + strsect->sh_name);
154
149
155
150
/* We'll tack temporary mod_kallsyms on the end. */
156
-
mod->init_layout.size = ALIGN(mod->init_layout.size,
157
-
__alignof__(struct mod_kallsyms));
158
-
info->mod_kallsyms_init_off = mod->init_layout.size;
159
-
mod->init_layout.size += sizeof(struct mod_kallsyms);
160
-
info->init_typeoffs = mod->init_layout.size;
161
-
mod->init_layout.size += nsrc * sizeof(char);
162
-
mod->init_layout.size = strict_align(mod->init_layout.size);
151
+
mod_mem_init_data->size = ALIGN(mod_mem_init_data->size,
152
+
__alignof__(struct mod_kallsyms));
153
+
info->mod_kallsyms_init_off = mod_mem_init_data->size;
154
+
155
+
mod_mem_init_data->size += sizeof(struct mod_kallsyms);
156
+
info->init_typeoffs = mod_mem_init_data->size;
157
+
mod_mem_init_data->size += nsrc * sizeof(char);
163
158
}
164
159
165
160
/*
···
174
171
char *s;
175
172
Elf_Shdr *symsec = &info->sechdrs[info->index.sym];
176
173
unsigned long strtab_size;
174
+
void *data_base = mod->mem[MOD_DATA].base;
175
+
void *init_data_base = mod->mem[MOD_INIT_DATA].base;
177
176
178
177
/* Set up to point into init section. */
179
-
mod->kallsyms = (void __rcu *)mod->init_layout.base +
178
+
mod->kallsyms = (void __rcu *)init_data_base +
180
179
info->mod_kallsyms_init_off;
181
180
182
181
rcu_read_lock();
···
188
183
/* Make sure we get permanent strtab: don't use info->strtab. */
189
184
rcu_dereference(mod->kallsyms)->strtab =
190
185
(void *)info->sechdrs[info->index.str].sh_addr;
191
-
rcu_dereference(mod->kallsyms)->typetab = mod->init_layout.base + info->init_typeoffs;
186
+
rcu_dereference(mod->kallsyms)->typetab = init_data_base + info->init_typeoffs;
192
187
193
188
/*
194
189
* Now populate the cut down core kallsyms for after init
195
190
* and set types up while we still have access to sections.
196
191
*/
197
-
mod->core_kallsyms.symtab = dst = mod->data_layout.base + info->symoffs;
198
-
mod->core_kallsyms.strtab = s = mod->data_layout.base + info->stroffs;
199
-
mod->core_kallsyms.typetab = mod->data_layout.base + info->core_typeoffs;
192
+
mod->core_kallsyms.symtab = dst = data_base + info->symoffs;
193
+
mod->core_kallsyms.strtab = s = data_base + info->stroffs;
194
+
mod->core_kallsyms.typetab = data_base + info->core_typeoffs;
200
195
strtab_size = info->core_typeoffs - info->stroffs;
201
196
src = rcu_dereference(mod->kallsyms)->symtab;
202
197
for (ndst = i = 0; i < rcu_dereference(mod->kallsyms)->num_symtab; i++) {
···
272
267
unsigned int i, best = 0;
273
268
unsigned long nextval, bestval;
274
269
struct mod_kallsyms *kallsyms = rcu_dereference_sched(mod->kallsyms);
270
+
struct module_memory *mod_mem;
275
271
276
272
/* At worse, next value is at end of module */
277
273
if (within_module_init(addr, mod))
278
-
nextval = (unsigned long)mod->init_layout.base + mod->init_layout.text_size;
274
+
mod_mem = &mod->mem[MOD_INIT_TEXT];
279
275
else
280
-
nextval = (unsigned long)mod->core_layout.base + mod->core_layout.text_size;
276
+
mod_mem = &mod->mem[MOD_TEXT];
277
+
278
+
nextval = (unsigned long)mod_mem->base + mod_mem->size;
281
279
282
280
bestval = kallsyms_symbol_value(&kallsyms->symtab[best]);
283
281
+9
-8
kernel/module/kdb.c
+9
-8
kernel/module/kdb.c
···
26
26
if (mod->state == MODULE_STATE_UNFORMED)
27
27
continue;
28
28
29
-
kdb_printf("%-20s%8u", mod->name, mod->core_layout.size);
30
-
#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
31
-
kdb_printf("/%8u", mod->data_layout.size);
32
-
#endif
29
+
kdb_printf("%-20s%8u", mod->name, mod->mem[MOD_TEXT].size);
30
+
kdb_printf("/%8u", mod->mem[MOD_RODATA].size);
31
+
kdb_printf("/%8u", mod->mem[MOD_RO_AFTER_INIT].size);
32
+
kdb_printf("/%8u", mod->mem[MOD_DATA].size);
33
+
33
34
kdb_printf(" 0x%px ", (void *)mod);
34
35
#ifdef CONFIG_MODULE_UNLOAD
35
36
kdb_printf("%4d ", module_refcount(mod));
···
41
40
kdb_printf(" (Loading)");
42
41
else
43
42
kdb_printf(" (Live)");
44
-
kdb_printf(" 0x%px", mod->core_layout.base);
45
-
#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
46
-
kdb_printf("/0x%px", mod->data_layout.base);
47
-
#endif
43
+
kdb_printf(" 0x%px", mod->mem[MOD_TEXT].base);
44
+
kdb_printf("/0x%px", mod->mem[MOD_RODATA].base);
45
+
kdb_printf("/0x%px", mod->mem[MOD_RO_AFTER_INIT].base);
46
+
kdb_printf("/0x%px", mod->mem[MOD_DATA].base);
48
47
49
48
#ifdef CONFIG_MODULE_UNLOAD
50
49
{
+199
-196
kernel/module/main.c
+199
-196
kernel/module/main.c
···
80
80
.addr_min = -1UL,
81
81
};
82
82
83
-
#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
84
-
struct mod_tree_root mod_data_tree __cacheline_aligned = {
85
-
.addr_min = -1UL,
86
-
};
87
-
#endif
88
-
89
83
struct symsearch {
90
84
const struct kernel_symbol *start, *stop;
91
85
const s32 *crcs;
···
87
93
};
88
94
89
95
/*
90
-
* Bounds of module text, for speeding up __module_address.
96
+
* Bounds of module memory, for speeding up __module_address.
91
97
* Protected by module_mutex.
92
98
*/
93
-
static void __mod_update_bounds(void *base, unsigned int size, struct mod_tree_root *tree)
99
+
static void __mod_update_bounds(enum mod_mem_type type __maybe_unused, void *base,
100
+
unsigned int size, struct mod_tree_root *tree)
94
101
{
95
102
unsigned long min = (unsigned long)base;
96
103
unsigned long max = min + size;
97
104
105
+
#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
106
+
if (mod_mem_type_is_core_data(type)) {
107
+
if (min < tree->data_addr_min)
108
+
tree->data_addr_min = min;
109
+
if (max > tree->data_addr_max)
110
+
tree->data_addr_max = max;
111
+
return;
112
+
}
113
+
#endif
98
114
if (min < tree->addr_min)
99
115
tree->addr_min = min;
100
116
if (max > tree->addr_max)
···
113
109
114
110
static void mod_update_bounds(struct module *mod)
115
111
{
116
-
__mod_update_bounds(mod->core_layout.base, mod->core_layout.size, &mod_tree);
117
-
if (mod->init_layout.size)
118
-
__mod_update_bounds(mod->init_layout.base, mod->init_layout.size, &mod_tree);
119
-
#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
120
-
__mod_update_bounds(mod->data_layout.base, mod->data_layout.size, &mod_data_tree);
121
-
#endif
112
+
for_each_mod_mem_type(type) {
113
+
struct module_memory *mod_mem = &mod->mem[type];
114
+
115
+
if (mod_mem->size)
116
+
__mod_update_bounds(type, mod_mem->base, mod_mem->size, &mod_tree);
117
+
}
122
118
}
123
119
124
120
/* Block module loading/unloading? */
···
930
926
static ssize_t show_coresize(struct module_attribute *mattr,
931
927
struct module_kobject *mk, char *buffer)
932
928
{
933
-
return sprintf(buffer, "%u\n", mk->mod->core_layout.size);
929
+
unsigned int size = mk->mod->mem[MOD_TEXT].size;
930
+
931
+
if (!IS_ENABLED(CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC)) {
932
+
for_class_mod_mem_type(type, core_data)
933
+
size += mk->mod->mem[type].size;
934
+
}
935
+
return sprintf(buffer, "%u\n", size);
934
936
}
935
937
936
938
static struct module_attribute modinfo_coresize =
···
946
936
static ssize_t show_datasize(struct module_attribute *mattr,
947
937
struct module_kobject *mk, char *buffer)
948
938
{
949
-
return sprintf(buffer, "%u\n", mk->mod->data_layout.size);
939
+
unsigned int size = 0;
940
+
941
+
for_class_mod_mem_type(type, core_data)
942
+
size += mk->mod->mem[type].size;
943
+
return sprintf(buffer, "%u\n", size);
950
944
}
951
945
952
946
static struct module_attribute modinfo_datasize =
···
960
946
static ssize_t show_initsize(struct module_attribute *mattr,
961
947
struct module_kobject *mk, char *buffer)
962
948
{
963
-
return sprintf(buffer, "%u\n", mk->mod->init_layout.size);
949
+
unsigned int size = 0;
950
+
951
+
for_class_mod_mem_type(type, init)
952
+
size += mk->mod->mem[type].size;
953
+
return sprintf(buffer, "%u\n", size);
964
954
}
965
955
966
956
static struct module_attribute modinfo_initsize =
···
1161
1143
{
1162
1144
}
1163
1145
1146
+
static bool mod_mem_use_vmalloc(enum mod_mem_type type)
1147
+
{
1148
+
return IS_ENABLED(CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC) &&
1149
+
mod_mem_type_is_core_data(type);
1150
+
}
1151
+
1152
+
static void *module_memory_alloc(unsigned int size, enum mod_mem_type type)
1153
+
{
1154
+
if (mod_mem_use_vmalloc(type))
1155
+
return vzalloc(size);
1156
+
return module_alloc(size);
1157
+
}
1158
+
1159
+
static void module_memory_free(void *ptr, enum mod_mem_type type)
1160
+
{
1161
+
if (mod_mem_use_vmalloc(type))
1162
+
vfree(ptr);
1163
+
else
1164
+
module_memfree(ptr);
1165
+
}
1166
+
1167
+
static void free_mod_mem(struct module *mod)
1168
+
{
1169
+
for_each_mod_mem_type(type) {
1170
+
struct module_memory *mod_mem = &mod->mem[type];
1171
+
1172
+
if (type == MOD_DATA)
1173
+
continue;
1174
+
1175
+
/* Free lock-classes; relies on the preceding sync_rcu(). */
1176
+
lockdep_free_key_range(mod_mem->base, mod_mem->size);
1177
+
if (mod_mem->size)
1178
+
module_memory_free(mod_mem->base, type);
1179
+
}
1180
+
1181
+
/* MOD_DATA hosts mod, so free it at last */
1182
+
lockdep_free_key_range(mod->mem[MOD_DATA].base, mod->mem[MOD_DATA].size);
1183
+
module_memory_free(mod->mem[MOD_DATA].base, MOD_DATA);
1184
+
}
1185
+
1164
1186
/* Free a module, remove from lists, etc. */
1165
1187
static void free_module(struct module *mod)
1166
1188
{
···
1247
1189
1248
1190
/* This may be empty, but that's OK */
1249
1191
module_arch_freeing_init(mod);
1250
-
module_memfree(mod->init_layout.base);
1251
1192
kfree(mod->args);
1252
1193
percpu_modfree(mod);
1253
1194
1254
-
/* Free lock-classes; relies on the preceding sync_rcu(). */
1255
-
lockdep_free_key_range(mod->data_layout.base, mod->data_layout.size);
1256
-
1257
-
/* Finally, free the core (containing the module structure) */
1258
-
module_memfree(mod->core_layout.base);
1259
-
#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
1260
-
vfree(mod->data_layout.base);
1261
-
#endif
1195
+
free_mod_mem(mod);
1262
1196
}
1263
1197
1264
1198
void *__symbol_get(const char *symbol)
···
1437
1387
return 0;
1438
1388
}
1439
1389
1440
-
/* Update size with this section: return offset. */
1441
-
long module_get_offset(struct module *mod, unsigned int *size,
1442
-
Elf_Shdr *sechdr, unsigned int section)
1390
+
long module_get_offset_and_type(struct module *mod, enum mod_mem_type type,
1391
+
Elf_Shdr *sechdr, unsigned int section)
1443
1392
{
1444
-
long ret;
1393
+
long offset;
1394
+
long mask = ((unsigned long)(type) & SH_ENTSIZE_TYPE_MASK) << SH_ENTSIZE_TYPE_SHIFT;
1445
1395
1446
-
*size += arch_mod_section_prepend(mod, section);
1447
-
ret = ALIGN(*size, sechdr->sh_addralign ?: 1);
1448
-
*size = ret + sechdr->sh_size;
1449
-
return ret;
1396
+
mod->mem[type].size += arch_mod_section_prepend(mod, section);
1397
+
offset = ALIGN(mod->mem[type].size, sechdr->sh_addralign ?: 1);
1398
+
mod->mem[type].size = offset + sechdr->sh_size;
1399
+
1400
+
WARN_ON_ONCE(offset & mask);
1401
+
return offset | mask;
1450
1402
}
1451
1403
1452
1404
static bool module_init_layout_section(const char *sname)
···
1460
1408
return module_init_section(sname);
1461
1409
}
1462
1410
1463
-
/*
1464
-
* Lay out the SHF_ALLOC sections in a way not dissimilar to how ld
1465
-
* might -- code, read-only data, read-write data, small data. Tally
1466
-
* sizes, and place the offsets into sh_entsize fields: high bit means it
1467
-
* belongs in init.
1468
-
*/
1469
-
static void layout_sections(struct module *mod, struct load_info *info)
1411
+
static void __layout_sections(struct module *mod, struct load_info *info, bool is_init)
1470
1412
{
1471
-
static unsigned long const masks[][2] = {
1413
+
unsigned int m, i;
1414
+
1415
+
static const unsigned long masks[][2] = {
1472
1416
/*
1473
1417
* NOTE: all executable code must be the first section
1474
1418
* in this array; otherwise modify the text_size
···
1476
1428
{ SHF_WRITE | SHF_ALLOC, ARCH_SHF_SMALL },
1477
1429
{ ARCH_SHF_SMALL | SHF_ALLOC, 0 }
1478
1430
};
1479
-
unsigned int m, i;
1431
+
static const int core_m_to_mem_type[] = {
1432
+
MOD_TEXT,
1433
+
MOD_RODATA,
1434
+
MOD_RO_AFTER_INIT,
1435
+
MOD_DATA,
1436
+
MOD_INVALID, /* This is needed to match the masks array */
1437
+
};
1438
+
static const int init_m_to_mem_type[] = {
1439
+
MOD_INIT_TEXT,
1440
+
MOD_INIT_RODATA,
1441
+
MOD_INVALID,
1442
+
MOD_INIT_DATA,
1443
+
MOD_INVALID, /* This is needed to match the masks array */
1444
+
};
1445
+
1446
+
for (m = 0; m < ARRAY_SIZE(masks); ++m) {
1447
+
enum mod_mem_type type = is_init ? init_m_to_mem_type[m] : core_m_to_mem_type[m];
1448
+
1449
+
for (i = 0; i < info->hdr->e_shnum; ++i) {
1450
+
Elf_Shdr *s = &info->sechdrs[i];
1451
+
const char *sname = info->secstrings + s->sh_name;
1452
+
1453
+
if ((s->sh_flags & masks[m][0]) != masks[m][0]
1454
+
|| (s->sh_flags & masks[m][1])
1455
+
|| s->sh_entsize != ~0UL
1456
+
|| is_init != module_init_layout_section(sname))
1457
+
continue;
1458
+
1459
+
if (WARN_ON_ONCE(type == MOD_INVALID))
1460
+
continue;
1461
+
1462
+
s->sh_entsize = module_get_offset_and_type(mod, type, s, i);
1463
+
pr_debug("\t%s\n", sname);
1464
+
}
1465
+
}
1466
+
}
1467
+
1468
+
/*
1469
+
* Lay out the SHF_ALLOC sections in a way not dissimilar to how ld
1470
+
* might -- code, read-only data, read-write data, small data. Tally
1471
+
* sizes, and place the offsets into sh_entsize fields: high bit means it
1472
+
* belongs in init.
1473
+
*/
1474
+
static void layout_sections(struct module *mod, struct load_info *info)
1475
+
{
1476
+
unsigned int i;
1480
1477
1481
1478
for (i = 0; i < info->hdr->e_shnum; i++)
1482
1479
info->sechdrs[i].sh_entsize = ~0UL;
1483
1480
1484
1481
pr_debug("Core section allocation order:\n");
1485
-
for (m = 0; m < ARRAY_SIZE(masks); ++m) {
1486
-
for (i = 0; i < info->hdr->e_shnum; ++i) {
1487
-
Elf_Shdr *s = &info->sechdrs[i];
1488
-
const char *sname = info->secstrings + s->sh_name;
1489
-
unsigned int *sizep;
1490
-
1491
-
if ((s->sh_flags & masks[m][0]) != masks[m][0]
1492
-
|| (s->sh_flags & masks[m][1])
1493
-
|| s->sh_entsize != ~0UL
1494
-
|| module_init_layout_section(sname))
1495
-
continue;
1496
-
sizep = m ? &mod->data_layout.size : &mod->core_layout.size;
1497
-
s->sh_entsize = module_get_offset(mod, sizep, s, i);
1498
-
pr_debug("\t%s\n", sname);
1499
-
}
1500
-
switch (m) {
1501
-
case 0: /* executable */
1502
-
mod->core_layout.size = strict_align(mod->core_layout.size);
1503
-
mod->core_layout.text_size = mod->core_layout.size;
1504
-
break;
1505
-
case 1: /* RO: text and ro-data */
1506
-
mod->data_layout.size = strict_align(mod->data_layout.size);
1507
-
mod->data_layout.ro_size = mod->data_layout.size;
1508
-
break;
1509
-
case 2: /* RO after init */
1510
-
mod->data_layout.size = strict_align(mod->data_layout.size);
1511
-
mod->data_layout.ro_after_init_size = mod->data_layout.size;
1512
-
break;
1513
-
case 4: /* whole core */
1514
-
mod->data_layout.size = strict_align(mod->data_layout.size);
1515
-
break;
1516
-
}
1517
-
}
1482
+
__layout_sections(mod, info, false);
1518
1483
1519
1484
pr_debug("Init section allocation order:\n");
1520
-
for (m = 0; m < ARRAY_SIZE(masks); ++m) {
1521
-
for (i = 0; i < info->hdr->e_shnum; ++i) {
1522
-
Elf_Shdr *s = &info->sechdrs[i];
1523
-
const char *sname = info->secstrings + s->sh_name;
1524
-
1525
-
if ((s->sh_flags & masks[m][0]) != masks[m][0]
1526
-
|| (s->sh_flags & masks[m][1])
1527
-
|| s->sh_entsize != ~0UL
1528
-
|| !module_init_layout_section(sname))
1529
-
continue;
1530
-
s->sh_entsize = (module_get_offset(mod, &mod->init_layout.size, s, i)
1531
-
| INIT_OFFSET_MASK);
1532
-
pr_debug("\t%s\n", sname);
1533
-
}
1534
-
switch (m) {
1535
-
case 0: /* executable */
1536
-
mod->init_layout.size = strict_align(mod->init_layout.size);
1537
-
mod->init_layout.text_size = mod->init_layout.size;
1538
-
break;
1539
-
case 1: /* RO: text and ro-data */
1540
-
mod->init_layout.size = strict_align(mod->init_layout.size);
1541
-
mod->init_layout.ro_size = mod->init_layout.size;
1542
-
break;
1543
-
case 2:
1544
-
/*
1545
-
* RO after init doesn't apply to init_layout (only
1546
-
* core_layout), so it just takes the value of ro_size.
1547
-
*/
1548
-
mod->init_layout.ro_after_init_size = mod->init_layout.ro_size;
1549
-
break;
1550
-
case 4: /* whole init */
1551
-
mod->init_layout.size = strict_align(mod->init_layout.size);
1552
-
break;
1553
-
}
1554
-
}
1485
+
__layout_sections(mod, info, true);
1555
1486
}
1556
1487
1557
1488
static void set_license(struct module *mod, const char *license)
···
2149
2122
{
2150
2123
int i;
2151
2124
void *ptr;
2125
+
enum mod_mem_type t;
2152
2126
2153
-
/* Do the allocs. */
2154
-
ptr = module_alloc(mod->core_layout.size);
2155
-
/*
2156
-
* The pointer to this block is stored in the module structure
2157
-
* which is inside the block. Just mark it as not being a
2158
-
* leak.
2159
-
*/
2160
-
kmemleak_not_leak(ptr);
2161
-
if (!ptr)
2162
-
return -ENOMEM;
2127
+
for_each_mod_mem_type(type) {
2128
+
if (!mod->mem[type].size) {
2129
+
mod->mem[type].base = NULL;
2130
+
continue;
2131
+
}
2132
+
mod->mem[type].size = PAGE_ALIGN(mod->mem[type].size);
2133
+
ptr = module_memory_alloc(mod->mem[type].size, type);
2163
2134
2164
-
memset(ptr, 0, mod->core_layout.size);
2165
-
mod->core_layout.base = ptr;
2166
-
2167
-
if (mod->init_layout.size) {
2168
-
ptr = module_alloc(mod->init_layout.size);
2169
2135
/*
2170
2136
* The pointer to this block is stored in the module structure
2171
-
* which is inside the block. This block doesn't need to be
2172
-
* scanned as it contains data and code that will be freed
2173
-
* after the module is initialized.
2137
+
* which is inside the block. Just mark it as not being a
2138
+
* leak.
2174
2139
*/
2175
2140
kmemleak_ignore(ptr);
2176
2141
if (!ptr) {
2177
-
module_memfree(mod->core_layout.base);
2178
-
return -ENOMEM;
2142
+
t = type;
2143
+
goto out_enomem;
2179
2144
}
2180
-
memset(ptr, 0, mod->init_layout.size);
2181
-
mod->init_layout.base = ptr;
2182
-
} else
2183
-
mod->init_layout.base = NULL;
2184
-
2185
-
#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
2186
-
/* Do the allocs. */
2187
-
ptr = vzalloc(mod->data_layout.size);
2188
-
/*
2189
-
* The pointer to this block is stored in the module structure
2190
-
* which is inside the block. Just mark it as not being a
2191
-
* leak.
2192
-
*/
2193
-
kmemleak_not_leak(ptr);
2194
-
if (!ptr) {
2195
-
module_memfree(mod->core_layout.base);
2196
-
module_memfree(mod->init_layout.base);
2197
-
return -ENOMEM;
2145
+
memset(ptr, 0, mod->mem[type].size);
2146
+
mod->mem[type].base = ptr;
2198
2147
}
2199
2148
2200
-
mod->data_layout.base = ptr;
2201
-
#endif
2202
2149
/* Transfer each section which specifies SHF_ALLOC */
2203
2150
pr_debug("final section addresses:\n");
2204
2151
for (i = 0; i < info->hdr->e_shnum; i++) {
2205
2152
void *dest;
2206
2153
Elf_Shdr *shdr = &info->sechdrs[i];
2154
+
enum mod_mem_type type = shdr->sh_entsize >> SH_ENTSIZE_TYPE_SHIFT;
2207
2155
2208
2156
if (!(shdr->sh_flags & SHF_ALLOC))
2209
2157
continue;
2210
2158
2211
-
if (shdr->sh_entsize & INIT_OFFSET_MASK)
2212
-
dest = mod->init_layout.base
2213
-
+ (shdr->sh_entsize & ~INIT_OFFSET_MASK);
2214
-
else if (!(shdr->sh_flags & SHF_EXECINSTR))
2215
-
dest = mod->data_layout.base + shdr->sh_entsize;
2216
-
else
2217
-
dest = mod->core_layout.base + shdr->sh_entsize;
2159
+
dest = mod->mem[type].base + (shdr->sh_entsize & SH_ENTSIZE_OFFSET_MASK);
2218
2160
2219
2161
if (shdr->sh_type != SHT_NOBITS)
2220
2162
memcpy(dest, (void *)shdr->sh_addr, shdr->sh_size);
···
2194
2198
}
2195
2199
2196
2200
return 0;
2201
+
out_enomem:
2202
+
for (t--; t >= 0; t--)
2203
+
module_memory_free(mod->mem[t].base, t);
2204
+
return -ENOMEM;
2197
2205
}
2198
2206
2199
2207
static int check_module_license_and_versions(struct module *mod)
···
2242
2242
* Do it before processing of module parameters, so the module
2243
2243
* can provide parameter accessor functions of its own.
2244
2244
*/
2245
-
if (mod->init_layout.base)
2246
-
flush_icache_range((unsigned long)mod->init_layout.base,
2247
-
(unsigned long)mod->init_layout.base
2248
-
+ mod->init_layout.size);
2249
-
flush_icache_range((unsigned long)mod->core_layout.base,
2250
-
(unsigned long)mod->core_layout.base + mod->core_layout.size);
2245
+
for_each_mod_mem_type(type) {
2246
+
const struct module_memory *mod_mem = &mod->mem[type];
2247
+
2248
+
if (mod_mem->size) {
2249
+
flush_icache_range((unsigned long)mod_mem->base,
2250
+
(unsigned long)mod_mem->base + mod_mem->size);
2251
+
}
2252
+
}
2251
2253
}
2252
2254
2253
2255
bool __weak module_elf_check_arch(Elf_Ehdr *hdr)
···
2352
2350
{
2353
2351
percpu_modfree(mod);
2354
2352
module_arch_freeing_init(mod);
2355
-
module_memfree(mod->init_layout.base);
2356
-
module_memfree(mod->core_layout.base);
2357
-
#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
2358
-
vfree(mod->data_layout.base);
2359
-
#endif
2353
+
2354
+
free_mod_mem(mod);
2360
2355
}
2361
2356
2362
2357
int __weak module_finalize(const Elf_Ehdr *hdr,
···
2414
2415
/* For freeing module_init on success, in case kallsyms traversing */
2415
2416
struct mod_initfree {
2416
2417
struct llist_node node;
2417
-
void *module_init;
2418
+
void *init_text;
2419
+
void *init_data;
2420
+
void *init_rodata;
2418
2421
};
2419
2422
2420
2423
static void do_free_init(struct work_struct *w)
···
2430
2429
2431
2430
llist_for_each_safe(pos, n, list) {
2432
2431
initfree = container_of(pos, struct mod_initfree, node);
2433
-
module_memfree(initfree->module_init);
2432
+
module_memfree(initfree->init_text);
2433
+
module_memfree(initfree->init_data);
2434
+
module_memfree(initfree->init_rodata);
2434
2435
kfree(initfree);
2435
2436
}
2436
2437
}
···
2459
2456
ret = -ENOMEM;
2460
2457
goto fail;
2461
2458
}
2462
-
freeinit->module_init = mod->init_layout.base;
2459
+
freeinit->init_text = mod->mem[MOD_INIT_TEXT].base;
2460
+
freeinit->init_data = mod->mem[MOD_INIT_DATA].base;
2461
+
freeinit->init_rodata = mod->mem[MOD_INIT_RODATA].base;
2463
2462
2464
2463
do_mod_ctors(mod);
2465
2464
/* Start the module */
···
2497
2492
if (!mod->async_probe_requested)
2498
2493
async_synchronize_full();
2499
2494
2500
-
ftrace_free_mem(mod, mod->init_layout.base, mod->init_layout.base +
2501
-
mod->init_layout.size);
2495
+
ftrace_free_mem(mod, mod->mem[MOD_INIT_TEXT].base,
2496
+
mod->mem[MOD_INIT_TEXT].base + mod->mem[MOD_INIT_TEXT].size);
2502
2497
mutex_lock(&module_mutex);
2503
2498
/* Drop initial reference. */
2504
2499
module_put(mod);
···
2510
2505
module_enable_ro(mod, true);
2511
2506
mod_tree_remove_init(mod);
2512
2507
module_arch_freeing_init(mod);
2513
-
mod->init_layout.base = NULL;
2514
-
mod->init_layout.size = 0;
2515
-
mod->init_layout.ro_size = 0;
2516
-
mod->init_layout.ro_after_init_size = 0;
2517
-
mod->init_layout.text_size = 0;
2508
+
for_class_mod_mem_type(type, init) {
2509
+
mod->mem[type].base = NULL;
2510
+
mod->mem[type].size = 0;
2511
+
}
2518
2512
#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
2519
2513
/* .BTF is not SHF_ALLOC and will get removed, so sanitize pointer */
2520
2514
mod->btf_data = NULL;
···
2632
2628
module_bug_finalize(info->hdr, info->sechdrs, mod);
2633
2629
module_cfi_finalize(info->hdr, info->sechdrs, mod);
2634
2630
2635
-
if (module_check_misalignment(mod))
2636
-
goto out_misaligned;
2637
-
2638
2631
module_enable_ro(mod, false);
2639
2632
module_enable_nx(mod);
2640
2633
module_enable_x(mod);
···
2645
2644
2646
2645
return 0;
2647
2646
2648
-
out_misaligned:
2649
-
err = -EINVAL;
2650
2647
out:
2651
2648
mutex_unlock(&module_mutex);
2652
2649
return err;
···
2908
2909
mutex_unlock(&module_mutex);
2909
2910
free_module:
2910
2911
/* Free lock-classes; relies on the preceding sync_rcu() */
2911
-
lockdep_free_key_range(mod->data_layout.base, mod->data_layout.size);
2912
+
for_class_mod_mem_type(type, core_data) {
2913
+
lockdep_free_key_range(mod->mem[type].base,
2914
+
mod->mem[type].size);
2915
+
}
2912
2916
2913
2917
module_deallocate(mod, info);
2914
2918
free_copy:
···
3062
3060
struct module *__module_address(unsigned long addr)
3063
3061
{
3064
3062
struct module *mod;
3065
-
struct mod_tree_root *tree;
3066
3063
3067
3064
if (addr >= mod_tree.addr_min && addr <= mod_tree.addr_max)
3068
-
tree = &mod_tree;
3069
-
#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
3070
-
else if (addr >= mod_data_tree.addr_min && addr <= mod_data_tree.addr_max)
3071
-
tree = &mod_data_tree;
3072
-
#endif
3073
-
else
3074
-
return NULL;
3065
+
goto lookup;
3075
3066
3067
+
#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
3068
+
if (addr >= mod_tree.data_addr_min && addr <= mod_tree.data_addr_max)
3069
+
goto lookup;
3070
+
#endif
3071
+
3072
+
return NULL;
3073
+
3074
+
lookup:
3076
3075
module_assert_mutex_or_preempt();
3077
3076
3078
-
mod = mod_find(addr, tree);
3077
+
mod = mod_find(addr, &mod_tree);
3079
3078
if (mod) {
3080
3079
BUG_ON(!within_module(addr, mod));
3081
3080
if (mod->state == MODULE_STATE_UNFORMED)
···
3116
3113
struct module *mod = __module_address(addr);
3117
3114
if (mod) {
3118
3115
/* Make sure it's within the text section. */
3119
-
if (!within(addr, mod->init_layout.base, mod->init_layout.text_size)
3120
-
&& !within(addr, mod->core_layout.base, mod->core_layout.text_size))
3116
+
if (!within_module_mem_type(addr, mod, MOD_TEXT) &&
3117
+
!within_module_mem_type(addr, mod, MOD_INIT_TEXT))
3121
3118
mod = NULL;
3122
3119
}
3123
3120
return mod;
+11
-5
kernel/module/procfs.c
+11
-5
kernel/module/procfs.c
···
62
62
mutex_unlock(&module_mutex);
63
63
}
64
64
65
+
static unsigned int module_total_size(struct module *mod)
66
+
{
67
+
int size = 0;
68
+
69
+
for_each_mod_mem_type(type)
70
+
size += mod->mem[type].size;
71
+
return size;
72
+
}
73
+
65
74
static int m_show(struct seq_file *m, void *p)
66
75
{
67
76
struct module *mod = list_entry(p, struct module, list);
···
82
73
if (mod->state == MODULE_STATE_UNFORMED)
83
74
return 0;
84
75
85
-
size = mod->init_layout.size + mod->core_layout.size;
86
-
#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
87
-
size += mod->data_layout.size;
88
-
#endif
76
+
size = module_total_size(mod);
89
77
seq_printf(m, "%s %u", mod->name, size);
90
78
print_unload_info(m, mod);
91
79
···
92
86
mod->state == MODULE_STATE_COMING ? "Loading" :
93
87
"Live");
94
88
/* Used by oprofile and other similar tools. */
95
-
value = m->private ? NULL : mod->core_layout.base;
89
+
value = m->private ? NULL : mod->mem[MOD_TEXT].base;
96
90
seq_printf(m, " 0x%px", value);
97
91
98
92
/* Taints info */
+18
-81
kernel/module/strict_rwx.c
+18
-81
kernel/module/strict_rwx.c
···
11
11
#include <linux/set_memory.h>
12
12
#include "internal.h"
13
13
14
-
/*
15
-
* LKM RO/NX protection: protect module's text/ro-data
16
-
* from modification and any data from execution.
17
-
*
18
-
* General layout of module is:
19
-
* [text] [read-only-data] [ro-after-init] [writable data]
20
-
* text_size -----^ ^ ^ ^
21
-
* ro_size ------------------------| | |
22
-
* ro_after_init_size -----------------------------| |
23
-
* size -----------------------------------------------------------|
24
-
*
25
-
* These values are always page-aligned (as is base) when
26
-
* CONFIG_STRICT_MODULE_RWX is set.
27
-
*/
14
+
static void module_set_memory(const struct module *mod, enum mod_mem_type type,
15
+
int (*set_memory)(unsigned long start, int num_pages))
16
+
{
17
+
const struct module_memory *mod_mem = &mod->mem[type];
18
+
19
+
set_vm_flush_reset_perms(mod_mem->base);
20
+
set_memory((unsigned long)mod_mem->base, mod_mem->size >> PAGE_SHIFT);
21
+
}
28
22
29
23
/*
30
24
* Since some arches are moving towards PAGE_KERNEL module allocations instead
31
-
* of PAGE_KERNEL_EXEC, keep frob_text() and module_enable_x() independent of
25
+
* of PAGE_KERNEL_EXEC, keep module_enable_x() independent of
32
26
* CONFIG_STRICT_MODULE_RWX because they are needed regardless of whether we
33
27
* are strict.
34
28
*/
35
-
static void frob_text(const struct module_layout *layout,
36
-
int (*set_memory)(unsigned long start, int num_pages))
37
-
{
38
-
set_memory((unsigned long)layout->base,
39
-
PAGE_ALIGN(layout->text_size) >> PAGE_SHIFT);
40
-
}
41
-
42
-
static void frob_rodata(const struct module_layout *layout,
43
-
int (*set_memory)(unsigned long start, int num_pages))
44
-
{
45
-
set_memory((unsigned long)layout->base + layout->text_size,
46
-
(layout->ro_size - layout->text_size) >> PAGE_SHIFT);
47
-
}
48
-
49
-
static void frob_ro_after_init(const struct module_layout *layout,
50
-
int (*set_memory)(unsigned long start, int num_pages))
51
-
{
52
-
set_memory((unsigned long)layout->base + layout->ro_size,
53
-
(layout->ro_after_init_size - layout->ro_size) >> PAGE_SHIFT);
54
-
}
55
-
56
-
static void frob_writable_data(const struct module_layout *layout,
57
-
int (*set_memory)(unsigned long start, int num_pages))
58
-
{
59
-
set_memory((unsigned long)layout->base + layout->ro_after_init_size,
60
-
(layout->size - layout->ro_after_init_size) >> PAGE_SHIFT);
61
-
}
62
-
63
-
static bool layout_check_misalignment(const struct module_layout *layout)
64
-
{
65
-
return WARN_ON(!PAGE_ALIGNED(layout->base)) ||
66
-
WARN_ON(!PAGE_ALIGNED(layout->text_size)) ||
67
-
WARN_ON(!PAGE_ALIGNED(layout->ro_size)) ||
68
-
WARN_ON(!PAGE_ALIGNED(layout->ro_after_init_size)) ||
69
-
WARN_ON(!PAGE_ALIGNED(layout->size));
70
-
}
71
-
72
-
bool module_check_misalignment(const struct module *mod)
73
-
{
74
-
if (!IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
75
-
return false;
76
-
77
-
return layout_check_misalignment(&mod->core_layout) ||
78
-
layout_check_misalignment(&mod->data_layout) ||
79
-
layout_check_misalignment(&mod->init_layout);
80
-
}
81
-
82
29
void module_enable_x(const struct module *mod)
83
30
{
84
-
if (!PAGE_ALIGNED(mod->core_layout.base) ||
85
-
!PAGE_ALIGNED(mod->init_layout.base))
86
-
return;
87
-
88
-
frob_text(&mod->core_layout, set_memory_x);
89
-
frob_text(&mod->init_layout, set_memory_x);
31
+
for_class_mod_mem_type(type, text)
32
+
module_set_memory(mod, type, set_memory_x);
90
33
}
91
34
92
35
void module_enable_ro(const struct module *mod, bool after_init)
···
41
98
return;
42
99
#endif
43
100
44
-
set_vm_flush_reset_perms(mod->core_layout.base);
45
-
set_vm_flush_reset_perms(mod->init_layout.base);
46
-
frob_text(&mod->core_layout, set_memory_ro);
47
-
48
-
frob_rodata(&mod->data_layout, set_memory_ro);
49
-
frob_text(&mod->init_layout, set_memory_ro);
50
-
frob_rodata(&mod->init_layout, set_memory_ro);
101
+
module_set_memory(mod, MOD_TEXT, set_memory_ro);
102
+
module_set_memory(mod, MOD_INIT_TEXT, set_memory_ro);
103
+
module_set_memory(mod, MOD_RODATA, set_memory_ro);
104
+
module_set_memory(mod, MOD_INIT_RODATA, set_memory_ro);
51
105
52
106
if (after_init)
53
-
frob_ro_after_init(&mod->data_layout, set_memory_ro);
107
+
module_set_memory(mod, MOD_RO_AFTER_INIT, set_memory_ro);
54
108
}
55
109
56
110
void module_enable_nx(const struct module *mod)
···
55
115
if (!IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
56
116
return;
57
117
58
-
frob_rodata(&mod->data_layout, set_memory_nx);
59
-
frob_ro_after_init(&mod->data_layout, set_memory_nx);
60
-
frob_writable_data(&mod->data_layout, set_memory_nx);
61
-
frob_rodata(&mod->init_layout, set_memory_nx);
62
-
frob_writable_data(&mod->init_layout, set_memory_nx);
118
+
for_class_mod_mem_type(type, data)
119
+
module_set_memory(mod, type, set_memory_nx);
63
120
}
64
121
65
122
int module_enforce_rwx_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs,
+17
-22
kernel/module/tree_lookup.c
+17
-22
kernel/module/tree_lookup.c
···
21
21
22
22
static __always_inline unsigned long __mod_tree_val(struct latch_tree_node *n)
23
23
{
24
-
struct module_layout *layout = container_of(n, struct module_layout, mtn.node);
24
+
struct module_memory *mod_mem = container_of(n, struct module_memory, mtn.node);
25
25
26
-
return (unsigned long)layout->base;
26
+
return (unsigned long)mod_mem->base;
27
27
}
28
28
29
29
static __always_inline unsigned long __mod_tree_size(struct latch_tree_node *n)
30
30
{
31
-
struct module_layout *layout = container_of(n, struct module_layout, mtn.node);
31
+
struct module_memory *mod_mem = container_of(n, struct module_memory, mtn.node);
32
32
33
-
return (unsigned long)layout->size;
33
+
return (unsigned long)mod_mem->size;
34
34
}
35
35
36
36
static __always_inline bool
···
77
77
*/
78
78
void mod_tree_insert(struct module *mod)
79
79
{
80
-
mod->core_layout.mtn.mod = mod;
81
-
mod->init_layout.mtn.mod = mod;
82
-
83
-
__mod_tree_insert(&mod->core_layout.mtn, &mod_tree);
84
-
if (mod->init_layout.size)
85
-
__mod_tree_insert(&mod->init_layout.mtn, &mod_tree);
86
-
87
-
#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
88
-
mod->data_layout.mtn.mod = mod;
89
-
__mod_tree_insert(&mod->data_layout.mtn, &mod_data_tree);
90
-
#endif
80
+
for_each_mod_mem_type(type) {
81
+
mod->mem[type].mtn.mod = mod;
82
+
if (mod->mem[type].size)
83
+
__mod_tree_insert(&mod->mem[type].mtn, &mod_tree);
84
+
}
91
85
}
92
86
93
87
void mod_tree_remove_init(struct module *mod)
94
88
{
95
-
if (mod->init_layout.size)
96
-
__mod_tree_remove(&mod->init_layout.mtn, &mod_tree);
89
+
for_class_mod_mem_type(type, init) {
90
+
if (mod->mem[type].size)
91
+
__mod_tree_remove(&mod->mem[type].mtn, &mod_tree);
92
+
}
97
93
}
98
94
99
95
void mod_tree_remove(struct module *mod)
100
96
{
101
-
__mod_tree_remove(&mod->core_layout.mtn, &mod_tree);
102
-
mod_tree_remove_init(mod);
103
-
#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
104
-
__mod_tree_remove(&mod->data_layout.mtn, &mod_data_tree);
105
-
#endif
97
+
for_each_mod_mem_type(type) {
98
+
if (mod->mem[type].size)
99
+
__mod_tree_remove(&mod->mem[type].mtn, &mod_tree);
100
+
}
106
101
}
107
102
108
103
struct module *mod_find(unsigned long addr, struct mod_tree_root *tree)