Serenity Operating System
Kernel: Move kernel above the 3GB virtual address mark
The kernel and its static data structures are no longer identity-mapped
in the bottom 8MB of the address space, but instead move above 3GB.
The first 8MB above 3GB are pseudo-identity-mapped to the bottom 8MB of
the physical address space. But things don't have to stay this way!
Thanks to Jesse who made an earlier attempt at this, it was really easy
to get device drivers working once the page tables were in place! :^)
Fixes #734.
+325
-125
-1
Kernel/Arch/i386/APIC.cpp
-1
Kernel/Arch/i386/APIC.cpp
···
146
146
g_apic_base = apic_base.as_ptr();
147
147
148
148
// copy ap init code to P8000
149
-
MM.map_for_kernel(VirtualAddress(0x8000), PhysicalAddress(0x8000));
150
149
memcpy(reinterpret_cast<u8*>(0x8000), reinterpret_cast<const u8*>(apic_ap_start), apic_ap_start_size);
151
150
return true;
152
151
}
+120
-6
Kernel/Arch/i386/Boot/boot.S
+120
-6
Kernel/Arch/i386/Boot/boot.S
···
31
31
.skip 32768
32
32
stack_top:
33
33
34
-
.section .page_tables
34
+
.section .page_tables, "aw", @nobits
35
35
.align 4096
36
-
page_tables_start:
37
-
.skip 4096*9
36
+
.global boot_pdpt
37
+
boot_pdpt:
38
+
.skip 4096
39
+
.global boot_pd0
40
+
boot_pd0:
41
+
.skip 4096
42
+
.global boot_pd3
43
+
boot_pd3:
44
+
.skip 4096
45
+
.global boot_pd3_pde1023_pt
46
+
boot_pd3_pde1023_pt:
47
+
.skip 4096
38
48
39
49
.section .text
40
50
···
47
57
.extern multiboot_info_ptr
48
58
.type multiboot_info_ptr, @object
49
59
60
+
/*
61
+
construct the following (32-bit PAE) page table layout:
62
+
63
+
pdpt
64
+
65
+
0: boot_pd0 (0-1GB)
66
+
1: n/a (1-2GB)
67
+
2: n/a (2-3GB)
68
+
3: boot_pd3 (3-4GB)
69
+
70
+
boot_pd0 : 512 pde's
71
+
72
+
0: (0-2MB) (id 2MB page)
73
+
1: (2-4MB) (id 2MB page)
74
+
2: (4-6MB) (id 2MB page)
75
+
3: (6-8MB) (id 2MB page)
76
+
77
+
boot_pd3 : 512 pde's
78
+
79
+
0: boot_pd3_pde0 (3072-3074MB) (pseudo)
80
+
1: boot_pd3_pde1 (3074-3076MB) (pseudo)
81
+
2: boot_pd3_pde2 (3076-3078MB) (pseudo)
82
+
3: boot_pd3_pde3 (3078-3080MB) (pseudo)
83
+
4: boot_pd3_pde1023_pt (4094-4096MB) (for page table mappings)
84
+
*/
85
+
50
86
start:
51
87
cli
52
88
cld
53
89
54
-
mov $stack_top, %esp
90
+
/* clear pdpt */
91
+
movl $(boot_pdpt - 0xc0000000), %edi
92
+
movl $1024, %ecx
93
+
xorl %eax, %eax
94
+
rep stosl
95
+
96
+
/* set up pdpt[0] and pdpt[3] */
97
+
movl $(boot_pdpt - 0xc0000000), %edi
98
+
movl $((boot_pd0 - 0xc0000000) + 1), 0(%edi)
99
+
movl $((boot_pd3 - 0xc0000000) + 1), 24(%edi)
100
+
101
+
/* clear pd0 */
102
+
movl $(boot_pd0 - 0xc0000000), %edi
103
+
movl $1024, %ecx
104
+
xorl %eax, %eax
105
+
rep stosl
55
106
107
+
/* identity map bottom 8MB using 2MB pages (only PDE, no PTE) */
108
+
movl $4, %ecx
109
+
xorl %eax, %eax
110
+
movl $(boot_pd0 - 0xc0000000), %edi
111
+
1:
112
+
movl %eax, 0(%edi)
113
+
/* PS(2MB) + R/W + Present */
114
+
orl $0x83, 0(%edi)
115
+
116
+
addl $8, %edi
117
+
addl $(1048576 * 2), %eax
118
+
loop 1b
119
+
120
+
/* clear pd3 */
121
+
movl $(boot_pd3 - 0xc0000000), %edi
122
+
movl $1024, %ecx
123
+
xorl %eax, %eax
124
+
rep stosl
125
+
126
+
/* pseudo-identity map first 8MB above 3GB mark using 2MB pages again */
127
+
movl $4, %ecx
128
+
xorl %eax, %eax
129
+
movl $(boot_pd3 - 0xc0000000), %edi
130
+
1:
131
+
movl %eax, 0(%edi)
132
+
/* PS(2MB) + R/W + Present */
133
+
orl $0x83, 0(%edi)
134
+
135
+
addl $8, %edi
136
+
addl $(1048576 * 2), %eax
137
+
loop 1b
138
+
139
+
/* create an empty page table for the top 2MB at the 4GB mark */
140
+
movl $(boot_pd3 - 0xc0000000), %edi
141
+
movl $(boot_pd3_pde1023_pt - 0xc0000000), 4088(%edi)
142
+
orl $0x3, 4088(%edi)
143
+
movl $0, 4092(%edi)
144
+
145
+
/* point CR3 to PDPT */
146
+
movl $(boot_pdpt - 0xc0000000), %eax
147
+
movl %eax, %cr3
148
+
149
+
/* enable PAE + PSE */
150
+
movl %cr4, %eax
151
+
orl $0x60, %eax
152
+
movl %eax, %cr4
153
+
154
+
/* enable PG */
155
+
movl %cr0, %eax
156
+
orl $0x80000000, %eax
157
+
movl %eax, %cr0
158
+
159
+
/* jmp to an address above the 3GB mark */
160
+
push %cs
161
+
push $1f
162
+
retf
163
+
1:
164
+
165
+
movl %cr3, %eax
166
+
movl %eax, %cr3
167
+
168
+
/* set up initial stack and jump into C++ land */
169
+
mov $stack_top, %esp
56
170
and $-16, %esp
57
171
58
-
mov %ebx, multiboot_info_ptr
172
+
addl $0xc0000000, %ebx
173
+
movl %ebx, multiboot_info_ptr
59
174
60
-
pushl $page_tables_start
61
175
call init
62
176
add $4, %esp
63
177
+17
-2
Kernel/Arch/i386/CPU.cpp
+17
-2
Kernel/Arch/i386/CPU.cpp
···
146
146
kprintf("eax=%08x ebx=%08x ecx=%08x edx=%08x\n", regs.eax, regs.ebx, regs.ecx, regs.edx);
147
147
kprintf("ebp=%08x esp=%08x esi=%08x edi=%08x\n", regs.ebp, esp, regs.esi, regs.edi);
148
148
149
+
u32 cr0;
150
+
asm("movl %%cr0, %%eax"
151
+
: "=a"(cr0));
152
+
u32 cr2;
153
+
asm("movl %%cr2, %%eax"
154
+
: "=a"(cr2));
155
+
u32 cr3;
156
+
asm("movl %%cr3, %%eax"
157
+
: "=a"(cr3));
158
+
u32 cr4;
159
+
asm("movl %%cr4, %%eax"
160
+
: "=a"(cr4));
161
+
kprintf("cr0=%08x cr2=%08x cr3=%08x cr4=%08x\n", cr0, cr2, cr3, cr4);
162
+
149
163
if (current && current->process().validate_read((void*)regs.eip, 8)) {
150
164
SmapDisabler disabler;
151
165
u8* codeptr = (u8*)regs.eip;
···
221
235
void page_fault_handler(RegisterDump regs)
222
236
{
223
237
clac();
224
-
ASSERT(current);
238
+
//ASSERT(current);
225
239
226
240
u32 fault_address;
227
241
asm("movl %%cr2, %%eax"
···
232
246
: "=a"(fault_page_directory));
233
247
234
248
#ifdef PAGE_FAULT_DEBUG
235
-
dbgprintf("%s(%u): ring%u %s page fault in PD=%x, %s V%08x\n",
249
+
dbgprintf("%s(%u): ring%u %s page fault in PD=%x, %s%s V%08x\n",
236
250
current ? current->process().name().characters() : "(none)",
237
251
current ? current->pid() : 0,
238
252
regs.cs & 3,
239
253
regs.exception_code & 1 ? "PV" : "NP",
240
254
fault_page_directory,
255
+
regs.exception_code & 8 ? "reserved-bit " : "",
241
256
regs.exception_code & 2 ? "write" : "read",
242
257
fault_address);
243
258
#endif
+6
Kernel/Arch/i386/CPU.h
+6
Kernel/Arch/i386/CPU.h
···
95
95
m_raw |= value & 0xfffff000;
96
96
}
97
97
98
+
void clear() { m_raw = 0; }
99
+
98
100
u64 raw() const { return m_raw; }
99
101
void copy_from(Badge<PageDirectory>, const PageDirectoryEntry& other) { m_raw = other.m_raw; }
100
102
···
104
106
UserSupervisor = 1 << 2,
105
107
WriteThrough = 1 << 3,
106
108
CacheDisabled = 1 << 4,
109
+
Huge = 1 << 7,
107
110
Global = 1 << 8,
108
111
NoExecute = 0x8000000000000000ULL,
109
112
};
···
113
116
114
117
bool is_user_allowed() const { return raw() & UserSupervisor; }
115
118
void set_user_allowed(bool b) { set_bit(UserSupervisor, b); }
119
+
120
+
bool is_huge() const { return raw() & Huge; }
121
+
void set_huge(bool b) { set_bit(Huge, b); }
116
122
117
123
bool is_writable() const { return raw() & ReadWrite; }
118
124
void set_writable(bool b) { set_bit(ReadWrite, b); }
+13
-11
Kernel/Devices/PATAChannel.cpp
+13
-11
Kernel/Devices/PATAChannel.cpp
···
102
102
m_dma_enabled.resource() = true;
103
103
ProcFS::add_sys_bool("ide_dma", m_dma_enabled);
104
104
105
+
m_prdt_page = MM.allocate_supervisor_physical_page();
106
+
105
107
initialize(force_pio);
106
108
detect_disks();
107
109
}
···
131
133
132
134
// Let's try to set up DMA transfers.
133
135
PCI::enable_bus_mastering(m_pci_address);
134
-
m_prdt.end_of_table = 0x8000;
136
+
prdt().end_of_table = 0x8000;
135
137
m_bus_master_base = PCI::get_BAR4(m_pci_address) & 0xfffc;
136
138
m_dma_buffer_page = MM.allocate_supervisor_physical_page();
137
139
kprintf("PATAChannel: Bus master IDE: I/O @ %x\n", m_bus_master_base);
···
253
255
current->pid(), lba, count, outbuf);
254
256
#endif
255
257
256
-
m_prdt.offset = m_dma_buffer_page->paddr();
257
-
m_prdt.size = 512 * count;
258
+
prdt().offset = m_dma_buffer_page->paddr();
259
+
prdt().size = 512 * count;
258
260
259
-
ASSERT(m_prdt.size <= PAGE_SIZE);
261
+
ASSERT(prdt().size <= PAGE_SIZE);
260
262
261
263
// Stop bus master
262
264
IO::out8(m_bus_master_base, 0);
263
265
264
266
// Write the PRDT location
265
-
IO::out32(m_bus_master_base + 4, (u32)&m_prdt);
267
+
IO::out32(m_bus_master_base + 4, m_prdt_page->paddr().get());
266
268
267
269
// Turn on "Interrupt" and "Error" flag. The error flag should be cleared by hardware.
268
270
IO::out8(m_bus_master_base + 2, IO::in8(m_bus_master_base + 2) | 0x6);
···
310
312
if (m_device_error)
311
313
return false;
312
314
313
-
memcpy(outbuf, m_dma_buffer_page->paddr().as_ptr(), 512 * count);
315
+
memcpy(outbuf, m_dma_buffer_page->paddr().offset(0xc0000000).as_ptr(), 512 * count);
314
316
315
317
// I read somewhere that this may trigger a cache flush so let's do it.
316
318
IO::out8(m_bus_master_base + 2, IO::in8(m_bus_master_base + 2) | 0x6);
···
326
328
current->pid(), lba, count, inbuf);
327
329
#endif
328
330
329
-
m_prdt.offset = m_dma_buffer_page->paddr();
330
-
m_prdt.size = 512 * count;
331
+
prdt().offset = m_dma_buffer_page->paddr();
332
+
prdt().size = 512 * count;
331
333
332
-
memcpy(m_dma_buffer_page->paddr().as_ptr(), inbuf, 512 * count);
334
+
memcpy(m_dma_buffer_page->paddr().offset(0xc0000000).as_ptr(), inbuf, 512 * count);
333
335
334
-
ASSERT(m_prdt.size <= PAGE_SIZE);
336
+
ASSERT(prdt().size <= PAGE_SIZE);
335
337
336
338
// Stop bus master
337
339
IO::out8(m_bus_master_base, 0);
338
340
339
341
// Write the PRDT location
340
-
IO::out32(m_bus_master_base + 4, (u32)&m_prdt);
342
+
IO::out32(m_bus_master_base + 4, m_prdt_page->paddr().get());
341
343
342
344
// Turn on "Interrupt" and "Error" flag. The error flag should be cleared by hardware.
343
345
IO::out8(m_bus_master_base + 2, IO::in8(m_bus_master_base + 2) | 0x6);
+2
-1
Kernel/Devices/PATAChannel.h
+2
-1
Kernel/Devices/PATAChannel.h
···
65
65
WaitQueue m_irq_queue;
66
66
67
67
PCI::Address m_pci_address;
68
-
PhysicalRegionDescriptor m_prdt;
68
+
PhysicalRegionDescriptor& prdt() { return *reinterpret_cast<PhysicalRegionDescriptor*>(m_prdt_page->paddr().offset(0xc0000000).as_ptr()); }
69
+
RefPtr<PhysicalPage> m_prdt_page;
69
70
RefPtr<PhysicalPage> m_dma_buffer_page;
70
71
u16 m_bus_master_base { 0 };
71
72
Lockable<bool> m_dma_enabled;
+2
-2
Kernel/Heap/kmalloc.cpp
+2
-2
Kernel/Heap/kmalloc.cpp
···
20
20
size_t nchunk;
21
21
};
22
22
23
-
#define BASE_PHYSICAL (4 * MB)
23
+
#define BASE_PHYSICAL (0xc0000000 + (4 * MB))
24
24
#define CHUNK_SIZE 8
25
25
#define POOL_SIZE (3 * MB)
26
26
27
-
#define ETERNAL_BASE_PHYSICAL (2 * MB)
27
+
#define ETERNAL_BASE_PHYSICAL (0xc0000000 + (2 * MB))
28
28
#define ETERNAL_RANGE_SIZE (2 * MB)
29
29
30
30
static u8 alloc_map[POOL_SIZE / CHUNK_SIZE / 8];
+1
-1
Kernel/Makefile
+1
-1
Kernel/Makefile
···
123
123
SUBPROJECT_CXXFLAGS += -I../Toolchain/Local/i686-pc-serenity/include/c++/9.2.0/
124
124
SUBPROJECT_CXXFLAGS += -I../Toolchain/Local/i686-pc-serenity/include/c++/9.2.0/i686-pc-serenity/
125
125
126
-
LDFLAGS += -Ttext 0x100000 -Wl,-T linker.ld -nostdlib -lgcc -lstdc++ -g3
126
+
LDFLAGS += -Wl,-T linker.ld -nostdlib -lgcc -lstdc++ -g3
127
127
128
128
all: $(PROGRAM) $(MODULE_OBJS) kernel.map
129
129
+13
-6
Kernel/Net/E1000NetworkAdapter.cpp
+13
-6
Kernel/Net/E1000NetworkAdapter.cpp
···
234
234
m_rx_descriptors = (e1000_rx_desc*)ptr;
235
235
for (int i = 0; i < number_of_rx_descriptors; ++i) {
236
236
auto& descriptor = m_rx_descriptors[i];
237
-
descriptor.addr = (u64)kmalloc_eternal(8192 + 16);
237
+
auto addr = (u32)kmalloc_eternal(8192 + 16);
238
+
if (addr % 16)
239
+
addr = (addr + 16) - (addr % 16);
240
+
descriptor.addr = addr - 0xc0000000;
238
241
descriptor.status = 0;
239
242
}
240
243
241
-
out32(REG_RXDESCLO, ptr);
244
+
out32(REG_RXDESCLO, (u32)ptr - 0xc0000000);
242
245
out32(REG_RXDESCHI, 0);
243
246
out32(REG_RXDESCLEN, number_of_rx_descriptors * sizeof(e1000_rx_desc));
244
247
out32(REG_RXDESCHEAD, 0);
···
256
259
m_tx_descriptors = (e1000_tx_desc*)ptr;
257
260
for (int i = 0; i < number_of_tx_descriptors; ++i) {
258
261
auto& descriptor = m_tx_descriptors[i];
259
-
descriptor.addr = (u64)kmalloc_eternal(8192 + 16);
262
+
auto addr = (u32)kmalloc_eternal(8192 + 16);
263
+
if (addr % 16)
264
+
addr = (addr + 16) - (addr % 16);
265
+
descriptor.addr = addr - 0xc0000000;
260
266
descriptor.cmd = 0;
261
267
}
262
268
263
-
out32(REG_TXDESCLO, ptr);
269
+
out32(REG_TXDESCLO, (u32)ptr - 0xc0000000);
264
270
out32(REG_TXDESCHI, 0);
265
271
out32(REG_TXDESCLEN, number_of_tx_descriptors * sizeof(e1000_tx_desc));
266
272
out32(REG_TXDESCHEAD, 0);
···
348
354
#endif
349
355
auto& descriptor = m_tx_descriptors[tx_current];
350
356
ASSERT(length <= 8192);
351
-
memcpy((void*)descriptor.addr, data, length);
357
+
auto *vptr = (void*)(descriptor.addr + 0xc0000000);
358
+
memcpy(vptr, data, length);
352
359
descriptor.length = length;
353
360
descriptor.status = 0;
354
361
descriptor.cmd = CMD_EOP | CMD_IFCS | CMD_RS;
···
381
388
rx_current = (rx_current + 1) % number_of_rx_descriptors;
382
389
if (!(m_rx_descriptors[rx_current].status & 1))
383
390
break;
384
-
auto* buffer = (u8*)m_rx_descriptors[rx_current].addr;
391
+
auto* buffer = (u8*)(m_rx_descriptors[rx_current].addr + 0xc0000000);
385
392
u16 length = m_rx_descriptors[rx_current].length;
386
393
#ifdef E1000_DEBUG
387
394
kprintf("E1000: Received 1 packet @ %p (%u) bytes!\n", buffer, length);
+1
-1
Kernel/TTY/VirtualConsole.cpp
+1
-1
Kernel/TTY/VirtualConsole.cpp
+79
-56
Kernel/VM/MemoryManager.cpp
+79
-56
Kernel/VM/MemoryManager.cpp
···
21
21
return *s_the;
22
22
}
23
23
24
-
MemoryManager::MemoryManager(u32 physical_address_for_kernel_page_tables)
24
+
MemoryManager::MemoryManager()
25
25
{
26
-
m_kernel_page_directory = PageDirectory::create_at_fixed_address(PhysicalAddress(physical_address_for_kernel_page_tables));
27
-
for (size_t i = 0; i < 4; ++i) {
28
-
m_low_page_tables[i] = (PageTableEntry*)(physical_address_for_kernel_page_tables + PAGE_SIZE * (5 + i));
29
-
memset(m_low_page_tables[i], 0, PAGE_SIZE);
30
-
}
26
+
m_kernel_page_directory = PageDirectory::create_kernel_page_directory();
31
27
32
28
initialize_paging();
33
29
···
49
45
dbgprintf("MM: Kernel page directory @ %p\n", kernel_page_directory().cr3());
50
46
#endif
51
47
52
-
#ifdef MM_DEBUG
53
-
dbgprintf("MM: Protect against null dereferences\n");
54
-
#endif
55
-
// Make null dereferences crash.
56
-
map_protected(VirtualAddress(0), PAGE_SIZE);
57
-
58
-
#ifdef MM_DEBUG
59
-
dbgprintf("MM: Identity map bottom 8MB\n");
60
-
#endif
61
-
// The bottom 8 MB (except for the null page) are identity mapped & supervisor only.
62
-
// Every process shares these mappings.
63
-
create_identity_mapping(kernel_page_directory(), VirtualAddress(PAGE_SIZE), (8 * MB) - PAGE_SIZE);
64
-
65
-
// Disable execution from 0MB through 1MB (BIOS data, legacy things, ...)
66
-
if (g_cpu_supports_nx) {
67
-
for (size_t i = 0; i < (1 * MB); i += PAGE_SIZE) {
68
-
auto& pte = ensure_pte(kernel_page_directory(), VirtualAddress(i));
69
-
pte.set_execute_disabled(true);
70
-
}
71
-
// Disable execution from 2MB through 8MB (kmalloc, kmalloc_eternal, slabs, page tables, ...)
72
-
for (size_t i = 1; i < 4; ++i) {
73
-
auto& pte = kernel_page_directory().table().directory(0)[i];
74
-
pte.set_execute_disabled(true);
75
-
}
76
-
}
48
+
// Disable execution from 0MB through 2MB (BIOS data, legacy things, ...)
49
+
if (g_cpu_supports_nx)
50
+
quickmap_pd(kernel_page_directory(), 0)[0].set_execute_disabled(true);
77
51
52
+
#if 0
78
53
// Disable writing to the kernel text and rodata segments.
79
54
extern u32 start_of_kernel_text;
80
55
extern u32 start_of_kernel_data;
···
91
66
pte.set_execute_disabled(true);
92
67
}
93
68
}
94
-
95
-
// FIXME: We should move everything kernel-related above the 0xc0000000 virtual mark.
96
-
97
-
// Basic physical memory map:
98
-
// 0 -> 1 MB We're just leaving this alone for now.
99
-
// 1 -> 2 MB Kernel image.
100
-
// (last page before 2MB) Used by quickmap_page().
101
-
// 2 MB -> 4 MB kmalloc_eternal() space.
102
-
// 4 MB -> 7 MB kmalloc() space.
103
-
// 7 MB -> 8 MB Supervisor physical pages (available for allocation!)
104
-
// 8 MB -> MAX Userspace physical pages (available for allocation!)
69
+
#endif
105
70
106
71
// Basic virtual memory map:
107
72
// 0 -> 4 KB Null page (so nullptr dereferences crash!)
···
109
74
// 8 MB -> 3 GB Available to userspace.
110
75
// 3GB -> 4 GB Kernel-only virtual address space (>0xc0000000)
111
76
77
+
m_quickmap_addr = VirtualAddress(0xffe00000);
112
78
#ifdef MM_DEBUG
113
79
dbgprintf("MM: Quickmap will use %p\n", m_quickmap_addr.get());
114
80
#endif
115
-
m_quickmap_addr = VirtualAddress((2 * MB) - PAGE_SIZE);
116
81
117
82
RefPtr<PhysicalRegion> region;
118
83
bool region_is_super = false;
119
84
120
-
for (auto* mmap = (multiboot_memory_map_t*)multiboot_info_ptr->mmap_addr; (unsigned long)mmap < multiboot_info_ptr->mmap_addr + multiboot_info_ptr->mmap_length; mmap = (multiboot_memory_map_t*)((unsigned long)mmap + mmap->size + sizeof(mmap->size))) {
85
+
auto* mmap = (multiboot_memory_map_t*)(0xc0000000 + multiboot_info_ptr->mmap_addr);
86
+
for (; (unsigned long)mmap < (0xc0000000 + multiboot_info_ptr->mmap_addr) + (multiboot_info_ptr->mmap_length); mmap = (multiboot_memory_map_t*)((unsigned long)mmap + mmap->size + sizeof(mmap->size))) {
121
87
kprintf("MM: Multiboot mmap: base_addr = 0x%x%08x, length = 0x%x%08x, type = 0x%x\n",
122
88
(u32)(mmap->addr >> 32),
123
89
(u32)(mmap->addr & 0xffffffff),
···
221
187
222
188
if (g_cpu_supports_smap) {
223
189
// Turn on CR4.SMAP
190
+
kprintf("x86: Enabling SMAP\n");
224
191
asm volatile(
225
192
"mov %cr4, %eax\n"
226
193
"orl $0x200000, %eax\n"
···
261
228
u32 page_directory_index = (vaddr.get() >> 21) & 0x1ff;
262
229
u32 page_table_index = (vaddr.get() >> 12) & 0x1ff;
263
230
264
-
PageDirectoryEntry& pde = page_directory.table().directory(page_directory_table_index)[page_directory_index];
231
+
auto* pd = quickmap_pd(page_directory, page_directory_table_index);
232
+
PageDirectoryEntry& pde = pd[page_directory_index];
265
233
if (!pde.is_present()) {
266
234
#ifdef MM_DEBUG
267
235
dbgprintf("MM: PDE %u not present (requested for V%p), allocating\n", page_directory_index, vaddr.get());
268
236
#endif
269
-
if (page_directory_table_index == 0 && page_directory_index < 4) {
270
-
ASSERT(&page_directory == m_kernel_page_directory);
271
-
pde.set_page_table_base((u32)m_low_page_tables[page_directory_index]);
272
-
pde.set_user_allowed(false);
273
-
pde.set_present(true);
274
-
pde.set_writable(true);
275
-
pde.set_global(true);
237
+
if (page_directory_table_index == 3 && page_directory_index < 4) {
238
+
ASSERT_NOT_REACHED();
276
239
} else {
277
240
auto page_table = allocate_supervisor_physical_page();
278
241
#ifdef MM_DEBUG
···
292
255
page_directory.m_physical_pages.set(page_directory_index, move(page_table));
293
256
}
294
257
}
295
-
return pde.page_table_base()[page_table_index];
258
+
259
+
//if (&page_directory != &kernel_page_directory() && page_directory_table_index != 3) {
260
+
return quickmap_pt(PhysicalAddress((u32)pde.page_table_base()))[page_table_index];
261
+
//}
262
+
263
+
auto* phys_ptr = &pde.page_table_base()[page_table_index];
264
+
return *(PageTableEntry*)((u8*)phys_ptr + 0xc0000000);
296
265
}
297
266
298
267
void MemoryManager::map_protected(VirtualAddress vaddr, size_t length)
···
325
294
}
326
295
}
327
296
328
-
void MemoryManager::initialize(u32 physical_address_for_kernel_page_tables)
297
+
void MemoryManager::initialize()
329
298
{
330
-
s_the = new MemoryManager(physical_address_for_kernel_page_tables);
299
+
s_the = new MemoryManager;
331
300
}
332
301
333
302
Region* MemoryManager::kernel_region_from_vaddr(VirtualAddress vaddr)
···
349
318
return ®ion;
350
319
}
351
320
dbg() << process << " Couldn't find user region for " << vaddr;
321
+
if (auto* kreg = kernel_region_from_vaddr(vaddr)) {
322
+
dbg() << process << " OTOH, there is a kernel region: " << kreg->range() << ": " << kreg->name();
323
+
} else {
324
+
dbg() << process << " AND no kernel region either";
325
+
}
326
+
327
+
process.dump_regions();
328
+
329
+
kprintf("Kernel regions:\n");
330
+
kprintf("BEGIN END SIZE ACCESS NAME\n");
331
+
for (auto& region : MM.m_kernel_regions) {
332
+
kprintf("%08x -- %08x %08x %c%c%c%c%c%c %s\n",
333
+
region.vaddr().get(),
334
+
region.vaddr().offset(region.size() - 1).get(),
335
+
region.size(),
336
+
region.is_readable() ? 'R' : ' ',
337
+
region.is_writable() ? 'W' : ' ',
338
+
region.is_executable() ? 'X' : ' ',
339
+
region.is_shared() ? 'S' : ' ',
340
+
region.is_stack() ? 'T' : ' ',
341
+
region.vmobject().is_purgeable() ? 'P' : ' ',
342
+
region.name().characters());
343
+
}
352
344
return nullptr;
353
345
}
354
346
···
567
559
dbgprintf("MM: allocate_supervisor_physical_page vending P%p\n", page->paddr().get());
568
560
#endif
569
561
570
-
fast_u32_fill((u32*)page->paddr().as_ptr(), 0, PAGE_SIZE / sizeof(u32));
562
+
fast_u32_fill((u32*)page->paddr().offset(0xc0000000).as_ptr(), 0, PAGE_SIZE / sizeof(u32));
571
563
++m_super_physical_pages_used;
572
564
return page;
573
565
}
···
599
591
:
600
592
: "m"(*(char*)vaddr.get())
601
593
: "memory");
594
+
}
595
+
596
+
extern "C" PageTableEntry boot_pd3_pde1023_pt[1024];
597
+
598
+
PageDirectoryEntry* MemoryManager::quickmap_pd(PageDirectory& directory, size_t pdpt_index)
599
+
{
600
+
auto& pte = boot_pd3_pde1023_pt[4];
601
+
auto pd_paddr = directory.m_directory_pages[pdpt_index]->paddr();
602
+
if (pte.physical_page_base() != pd_paddr.as_ptr()) {
603
+
//dbgprintf("quickmap_pd: Mapping P%p at 0xffe04000 in pte @ %p\n", directory.m_directory_pages[pdpt_index]->paddr().as_ptr(), &pte);
604
+
pte.set_physical_page_base(pd_paddr.get());
605
+
pte.set_present(true);
606
+
pte.set_writable(true);
607
+
pte.set_user_allowed(false);
608
+
flush_tlb(VirtualAddress(0xffe04000));
609
+
}
610
+
return (PageDirectoryEntry*)0xffe04000;
611
+
}
612
+
613
+
PageTableEntry* MemoryManager::quickmap_pt(PhysicalAddress pt_paddr)
614
+
{
615
+
auto& pte = boot_pd3_pde1023_pt[8];
616
+
if (pte.physical_page_base() != pt_paddr.as_ptr()) {
617
+
//dbgprintf("quickmap_pt: Mapping P%p at 0xffe08000 in pte @ %p\n", pt_paddr.as_ptr(), &pte);
618
+
pte.set_physical_page_base(pt_paddr.get());
619
+
pte.set_present(true);
620
+
pte.set_writable(true);
621
+
pte.set_user_allowed(false);
622
+
flush_tlb(VirtualAddress(0xffe08000));
623
+
}
624
+
return (PageTableEntry*)0xffe08000;
602
625
}
603
626
604
627
void MemoryManager::map_for_kernel(VirtualAddress vaddr, PhysicalAddress paddr, bool cache_disabled)
+27
-2
Kernel/VM/MemoryManager.h
+27
-2
Kernel/VM/MemoryManager.h
···
20
20
21
21
#define PAGE_ROUND_UP(x) ((((u32)(x)) + PAGE_SIZE - 1) & (~(PAGE_SIZE - 1)))
22
22
23
+
template<typename T>
24
+
inline T* low_physical_to_virtual(T* physical)
25
+
{
26
+
return (T*)(((u8*)physical) + 0xc0000000);
27
+
}
28
+
29
+
inline u32 low_physical_to_virtual(u32 physical)
30
+
{
31
+
return physical + 0xc0000000;
32
+
}
33
+
34
+
template<typename T>
35
+
inline T* virtual_to_low_physical(T* physical)
36
+
{
37
+
return (T*)(((u8*)physical) - 0xc0000000);
38
+
}
39
+
40
+
inline u32 virtual_to_low_physical(u32 physical)
41
+
{
42
+
return physical - 0xc0000000;
43
+
}
44
+
23
45
class KBuffer;
24
46
class SynthFSInode;
25
47
···
38
60
public:
39
61
static MemoryManager& the();
40
62
41
-
static void initialize(u32 physical_address_for_kernel_page_tables);
63
+
static void initialize();
42
64
43
65
PageFaultResponse handle_page_fault(const PageFault&);
44
66
···
85
107
static const Region* region_from_vaddr(const Process&, VirtualAddress);
86
108
87
109
private:
88
-
MemoryManager(u32 physical_address_for_kernel_page_tables);
110
+
MemoryManager();
89
111
~MemoryManager();
90
112
91
113
enum class AccessSpace { Kernel, User };
···
115
137
RefPtr<PhysicalPage> find_free_user_physical_page();
116
138
u8* quickmap_page(PhysicalPage&);
117
139
void unquickmap_page();
140
+
141
+
PageDirectoryEntry* quickmap_pd(PageDirectory&, size_t pdpt_index);
142
+
PageTableEntry* quickmap_pt(PhysicalAddress);
118
143
119
144
PageDirectory& kernel_page_directory() { return *m_kernel_page_directory; }
120
145
+23
-22
Kernel/VM/PageDirectory.cpp
+23
-22
Kernel/VM/PageDirectory.cpp
···
4
4
#include <Kernel/VM/PageDirectory.h>
5
5
6
6
static const u32 userspace_range_base = 0x01000000;
7
-
static const u32 kernelspace_range_base = 0xc0000000;
7
+
static const u32 kernelspace_range_base = 0xc0800000;
8
8
9
9
static HashMap<u32, PageDirectory*>& cr3_map()
10
10
{
···
21
21
return cr3_map().get(cr3).value_or({});
22
22
}
23
23
24
-
PageDirectory::PageDirectory(PhysicalAddress paddr)
25
-
: m_range_allocator(VirtualAddress(0xc0000000), 0x3f000000)
24
+
extern "C" u32 boot_pdpt;
25
+
extern "C" u32 boot_pd0;
26
+
extern "C" u32 boot_pd3;
27
+
28
+
PageDirectory::PageDirectory()
29
+
: m_range_allocator(VirtualAddress(0xc0c00000), 0x3f000000)
26
30
{
27
-
m_directory_table = PhysicalPage::create(paddr, true, false);
28
-
m_directory_pages[0] = PhysicalPage::create(paddr.offset(PAGE_SIZE * 1), true, false);
29
-
m_directory_pages[1] = PhysicalPage::create(paddr.offset(PAGE_SIZE * 2), true, false);
30
-
m_directory_pages[2] = PhysicalPage::create(paddr.offset(PAGE_SIZE * 3), true, false);
31
-
m_directory_pages[3] = PhysicalPage::create(paddr.offset(PAGE_SIZE * 4), true, false);
32
-
33
-
table().raw[0] = (u64)m_directory_pages[0]->paddr().as_ptr() | 1;
34
-
table().raw[1] = (u64)m_directory_pages[1]->paddr().as_ptr() | 1;
35
-
table().raw[2] = (u64)m_directory_pages[2]->paddr().as_ptr() | 1;
36
-
table().raw[3] = (u64)m_directory_pages[3]->paddr().as_ptr() | 1;
37
-
38
-
InterruptDisabler disabler;
39
-
cr3_map().set(cr3(), this);
31
+
// Adopt the page tables already set up by boot.S
32
+
PhysicalAddress boot_pdpt_paddr(virtual_to_low_physical((u32)&boot_pdpt));
33
+
PhysicalAddress boot_pd0_paddr(virtual_to_low_physical((u32)&boot_pd0));
34
+
PhysicalAddress boot_pd3_paddr(virtual_to_low_physical((u32)&boot_pd3));
35
+
kprintf("MM: boot_pdpt @ P%p\n", boot_pdpt_paddr.get());
36
+
kprintf("MM: boot_pd0 @ P%p\n", boot_pd0_paddr.get());
37
+
kprintf("MM: boot_pd3 @ P%p\n", boot_pd3_paddr.get());
38
+
m_directory_table = PhysicalPage::create(boot_pdpt_paddr, true, false);
39
+
m_directory_pages[0] = PhysicalPage::create(boot_pd0_paddr, true, false);
40
+
m_directory_pages[3] = PhysicalPage::create(boot_pd3_paddr, true, false);
40
41
}
41
42
42
43
PageDirectory::PageDirectory(Process& process, const RangeAllocator* parent_range_allocator)
···
44
45
, m_range_allocator(parent_range_allocator ? RangeAllocator(*parent_range_allocator) : RangeAllocator(VirtualAddress(userspace_range_base), kernelspace_range_base - userspace_range_base))
45
46
{
46
47
// Set up a userspace page directory
47
-
48
48
m_directory_table = MM.allocate_supervisor_physical_page();
49
49
m_directory_pages[0] = MM.allocate_supervisor_physical_page();
50
50
m_directory_pages[1] = MM.allocate_supervisor_physical_page();
···
58
58
table().raw[3] = (u64)m_directory_pages[3]->paddr().as_ptr() | 1;
59
59
60
60
// Clone bottom 8 MB of mappings from kernel_page_directory
61
-
table().directory(0)[0].copy_from({}, MM.kernel_page_directory().table().directory(0)[0]);
62
-
table().directory(0)[1].copy_from({}, MM.kernel_page_directory().table().directory(0)[1]);
63
-
table().directory(0)[2].copy_from({}, MM.kernel_page_directory().table().directory(0)[2]);
64
-
table().directory(0)[3].copy_from({}, MM.kernel_page_directory().table().directory(0)[3]);
61
+
PageDirectoryEntry buffer[4];
62
+
auto* kernel_pd = MM.quickmap_pd(MM.kernel_page_directory(), 0);
63
+
memcpy(buffer, kernel_pd, sizeof(PageDirectoryEntry) * 4);
64
+
auto* new_pd = MM.quickmap_pd(*this, 0);
65
+
memcpy(new_pd, buffer, sizeof(PageDirectoryEntry) * 4);
65
66
66
67
InterruptDisabler disabler;
67
68
cr3_map().set(cr3(), this);
···
74
75
#endif
75
76
InterruptDisabler disabler;
76
77
cr3_map().remove(cr3());
77
-
}
78
+
}
+3
-3
Kernel/VM/PageDirectory.h
+3
-3
Kernel/VM/PageDirectory.h
···
16
16
{
17
17
return adopt(*new PageDirectory(process, parent_range_allocator));
18
18
}
19
-
static NonnullRefPtr<PageDirectory> create_at_fixed_address(PhysicalAddress paddr) { return adopt(*new PageDirectory(paddr)); }
19
+
static NonnullRefPtr<PageDirectory> create_kernel_page_directory() { return adopt(*new PageDirectory); }
20
20
static RefPtr<PageDirectory> find_by_cr3(u32);
21
21
22
22
~PageDirectory();
23
23
24
24
u32 cr3() const { return m_directory_table->paddr().get(); }
25
-
PageDirectoryPointerTable& table() { return *reinterpret_cast<PageDirectoryPointerTable*>(cr3()); }
25
+
PageDirectoryPointerTable& table() { return *reinterpret_cast<PageDirectoryPointerTable*>(0xc0000000 + cr3()); }
26
26
27
27
RangeAllocator& range_allocator() { return m_range_allocator; }
28
28
···
31
31
32
32
private:
33
33
PageDirectory(Process&, const RangeAllocator* parent_range_allocator);
34
-
explicit PageDirectory(PhysicalAddress);
34
+
PageDirectory();
35
35
36
36
Process* m_process { nullptr };
37
37
RangeAllocator m_range_allocator;
+1
-1
Kernel/VM/Region.cpp
+1
-1
Kernel/VM/Region.cpp
···
242
242
pte.set_execute_disabled(!is_executable());
243
243
pte.set_user_allowed(is_user_accessible());
244
244
#ifdef MM_DEBUG
245
-
dbg() << "MM: >> region map (PD=" << m_page_directory->cr3() << ", PTE=" << (void*)pte.raw() << "{" << &pte << "}) " << name() << " " << page_vaddr << " => " << physical_page->paddr() << " (@" << physical_page.ptr() << ")";
245
+
dbg() << "MM: >> region map (PD=" << m_page_directory->cr3() << ", PTE=" << (void*)pte.raw() << "{" << &pte << "}) " << name() << " " << page_vaddr << " => " << physical_page->paddr() << " (@" << physical_page.ptr() << ")";
246
246
#endif
247
247
}
248
248
MM.flush_tlb(page_vaddr);
+7
-4
Kernel/init.cpp
+7
-4
Kernel/init.cpp
···
229
229
extern u32 __stack_chk_guard;
230
230
u32 __stack_chk_guard;
231
231
232
-
extern "C" [[noreturn]] void init(u32 physical_address_for_kernel_page_tables)
232
+
extern "C" [[noreturn]] void init()
233
233
{
234
234
// this is only used one time, directly below here. we can't use this part
235
235
// of libc at this point in the boot process, or we'd just pull strstr in
···
247
247
// process on live hardware.
248
248
//
249
249
// note: it must be the first option in the boot cmdline.
250
-
if (multiboot_info_ptr->cmdline && bad_prefix_check(reinterpret_cast<const char*>(multiboot_info_ptr->cmdline), "serial_debug"))
250
+
u32 cmdline = low_physical_to_virtual(multiboot_info_ptr->cmdline);
251
+
if (cmdline && bad_prefix_check(reinterpret_cast<const char*>(cmdline), "serial_debug"))
251
252
set_serial_debug(true);
252
253
253
254
detect_cpu_features();
···
256
257
slab_alloc_init();
257
258
258
259
// must come after kmalloc_init because we use AK_MAKE_ETERNAL in KParams
259
-
new KParams(String(reinterpret_cast<const char*>(multiboot_info_ptr->cmdline)));
260
+
new KParams(String(reinterpret_cast<const char*>(cmdline)));
260
261
261
262
bool text_debug = KParams::the().has("text_debug");
262
263
bool complete_acpi_disable = KParams::the().has("noacpi");
263
264
bool dynamic_acpi_disable = KParams::the().has("noacpi_aml");
264
265
bool pci_mmio_disable = KParams::the().has("nopci_mmio");
265
266
266
-
MemoryManager::initialize(physical_address_for_kernel_page_tables);
267
+
complete_acpi_disable = true;
268
+
269
+
MemoryManager::initialize();
267
270
268
271
if (complete_acpi_disable) {
269
272
ACPIParser::initialize_limited();
+10
-6
Kernel/linker.ld
+10
-6
Kernel/linker.ld
···
2
2
3
3
SECTIONS
4
4
{
5
-
. = 0x100000;
5
+
. = 0xc0100000;
6
6
7
-
.text BLOCK(4K) : ALIGN(4K)
7
+
start_of_kernel_image = .;
8
+
9
+
.text ALIGN(4K) : AT (ADDR(.text) - 0xc0000000)
8
10
{
9
11
Arch/i386/Boot/boot.ao
10
12
*(.multiboot)
11
-
*(.page_tables)
12
13
start_of_kernel_text = .;
13
14
*(.text)
14
15
*(.text.startup)
15
16
end_of_kernel_text = .;
16
17
}
17
18
18
-
.rodata BLOCK(4K) : ALIGN(4K)
19
+
.rodata ALIGN(4K) : AT (ADDR(.rodata) - 0xc0000000)
19
20
{
20
21
start_ctors = .;
21
22
*(.ctors)
···
24
25
*(.rodata)
25
26
}
26
27
27
-
.data BLOCK(4K) : ALIGN(4K)
28
+
.data ALIGN(4K) : AT (ADDR(.data) - 0xc0000000)
28
29
{
29
30
start_of_kernel_data = .;
30
31
*(.data)
31
32
end_of_kernel_data = .;
32
33
}
33
34
34
-
.bss BLOCK(4K) : ALIGN(4K)
35
+
.bss ALIGN(4K) : AT (ADDR(.bss) - 0xc0000000)
35
36
{
36
37
start_of_kernel_bss = .;
38
+
*(page_tables)
37
39
*(COMMON)
38
40
*(.bss)
39
41
end_of_kernel_bss = .;
40
42
}
43
+
44
+
end_of_kernel_image = .;
41
45
}