Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
RISC-V: KVM: Implement stage2 page table programming
This patch implements all required functions for programming
the stage2 page table for each Guest/VM.
At high-level, the flow of stage2 related functions is similar
from KVM ARM/ARM64 implementation but the stage2 page table
format is quite different for KVM RISC-V.
[jiangyifei: stage2 dirty log support]
Signed-off-by: Yifei Jiang <jiangyifei@huawei.com>
Signed-off-by: Anup Patel <anup.patel@wdc.com>
Acked-by: Paolo Bonzini <pbonzini@redhat.com>
Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>
Acked-by: Palmer Dabbelt <palmerdabbelt@google.com>
authored by
Anup Patel
and committed by
Anup Patel
9d05c1fe
fd7bb4a2
+676
-16
+12
arch/riscv/include/asm/kvm_host.h
+12
arch/riscv/include/asm/kvm_host.h
···
70
70
int return_handled;
71
71
};
72
72
73
+
#define KVM_MMU_PAGE_CACHE_NR_OBJS 32
74
+
75
+
struct kvm_mmu_page_cache {
76
+
int nobjs;
77
+
void *objects[KVM_MMU_PAGE_CACHE_NR_OBJS];
78
+
};
79
+
73
80
struct kvm_cpu_trap {
74
81
unsigned long sepc;
75
82
unsigned long scause;
···
178
171
/* MMIO instruction details */
179
172
struct kvm_mmio_decode mmio_decode;
180
173
174
+
/* Cache pages needed to program page tables with spinlock held */
175
+
struct kvm_mmu_page_cache mmu_page_cache;
176
+
181
177
/* VCPU power-off state */
182
178
bool power_off;
183
179
···
208
198
int kvm_riscv_stage2_alloc_pgd(struct kvm *kvm);
209
199
void kvm_riscv_stage2_free_pgd(struct kvm *kvm);
210
200
void kvm_riscv_stage2_update_hgatp(struct kvm_vcpu *vcpu);
201
+
void kvm_riscv_stage2_mode_detect(void);
202
+
unsigned long kvm_riscv_stage2_mode(void);
211
203
212
204
void kvm_riscv_stage2_vmid_detect(void);
213
205
unsigned long kvm_riscv_stage2_vmid_bits(void);
+1
arch/riscv/kvm/Kconfig
+1
arch/riscv/kvm/Kconfig
+19
arch/riscv/kvm/main.c
+19
arch/riscv/kvm/main.c
···
64
64
65
65
int kvm_arch_init(void *opaque)
66
66
{
67
+
const char *str;
68
+
67
69
if (!riscv_isa_extension_available(NULL, h)) {
68
70
kvm_info("hypervisor extension not available\n");
69
71
return -ENODEV;
···
81
79
return -ENODEV;
82
80
}
83
81
82
+
kvm_riscv_stage2_mode_detect();
83
+
84
84
kvm_riscv_stage2_vmid_detect();
85
85
86
86
kvm_info("hypervisor extension available\n");
87
+
88
+
switch (kvm_riscv_stage2_mode()) {
89
+
case HGATP_MODE_SV32X4:
90
+
str = "Sv32x4";
91
+
break;
92
+
case HGATP_MODE_SV39X4:
93
+
str = "Sv39x4";
94
+
break;
95
+
case HGATP_MODE_SV48X4:
96
+
str = "Sv48x4";
97
+
break;
98
+
default:
99
+
return -ENODEV;
100
+
}
101
+
kvm_info("using %s G-stage page table format\n", str);
87
102
88
103
kvm_info("VMID %ld bits available\n", kvm_riscv_stage2_vmid_bits());
89
104
+644
-10
arch/riscv/kvm/mmu.c
+644
-10
arch/riscv/kvm/mmu.c
···
15
15
#include <linux/vmalloc.h>
16
16
#include <linux/kvm_host.h>
17
17
#include <linux/sched/signal.h>
18
+
#include <asm/csr.h>
18
19
#include <asm/page.h>
19
20
#include <asm/pgtable.h>
21
+
#include <asm/sbi.h>
22
+
23
+
#ifdef CONFIG_64BIT
24
+
static unsigned long stage2_mode = (HGATP_MODE_SV39X4 << HGATP_MODE_SHIFT);
25
+
static unsigned long stage2_pgd_levels = 3;
26
+
#define stage2_index_bits 9
27
+
#else
28
+
static unsigned long stage2_mode = (HGATP_MODE_SV32X4 << HGATP_MODE_SHIFT);
29
+
static unsigned long stage2_pgd_levels = 2;
30
+
#define stage2_index_bits 10
31
+
#endif
32
+
33
+
#define stage2_pgd_xbits 2
34
+
#define stage2_pgd_size (1UL << (HGATP_PAGE_SHIFT + stage2_pgd_xbits))
35
+
#define stage2_gpa_bits (HGATP_PAGE_SHIFT + \
36
+
(stage2_pgd_levels * stage2_index_bits) + \
37
+
stage2_pgd_xbits)
38
+
#define stage2_gpa_size ((gpa_t)(1ULL << stage2_gpa_bits))
39
+
40
+
#define stage2_pte_leaf(__ptep) \
41
+
(pte_val(*(__ptep)) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC))
42
+
43
+
static inline unsigned long stage2_pte_index(gpa_t addr, u32 level)
44
+
{
45
+
unsigned long mask;
46
+
unsigned long shift = HGATP_PAGE_SHIFT + (stage2_index_bits * level);
47
+
48
+
if (level == (stage2_pgd_levels - 1))
49
+
mask = (PTRS_PER_PTE * (1UL << stage2_pgd_xbits)) - 1;
50
+
else
51
+
mask = PTRS_PER_PTE - 1;
52
+
53
+
return (addr >> shift) & mask;
54
+
}
55
+
56
+
static inline unsigned long stage2_pte_page_vaddr(pte_t pte)
57
+
{
58
+
return (unsigned long)pfn_to_virt(pte_val(pte) >> _PAGE_PFN_SHIFT);
59
+
}
60
+
61
+
static int stage2_page_size_to_level(unsigned long page_size, u32 *out_level)
62
+
{
63
+
u32 i;
64
+
unsigned long psz = 1UL << 12;
65
+
66
+
for (i = 0; i < stage2_pgd_levels; i++) {
67
+
if (page_size == (psz << (i * stage2_index_bits))) {
68
+
*out_level = i;
69
+
return 0;
70
+
}
71
+
}
72
+
73
+
return -EINVAL;
74
+
}
75
+
76
+
static int stage2_level_to_page_size(u32 level, unsigned long *out_pgsize)
77
+
{
78
+
if (stage2_pgd_levels < level)
79
+
return -EINVAL;
80
+
81
+
*out_pgsize = 1UL << (12 + (level * stage2_index_bits));
82
+
83
+
return 0;
84
+
}
85
+
86
+
static int stage2_cache_topup(struct kvm_mmu_page_cache *pcache,
87
+
int min, int max)
88
+
{
89
+
void *page;
90
+
91
+
BUG_ON(max > KVM_MMU_PAGE_CACHE_NR_OBJS);
92
+
if (pcache->nobjs >= min)
93
+
return 0;
94
+
while (pcache->nobjs < max) {
95
+
page = (void *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
96
+
if (!page)
97
+
return -ENOMEM;
98
+
pcache->objects[pcache->nobjs++] = page;
99
+
}
100
+
101
+
return 0;
102
+
}
103
+
104
+
static void stage2_cache_flush(struct kvm_mmu_page_cache *pcache)
105
+
{
106
+
while (pcache && pcache->nobjs)
107
+
free_page((unsigned long)pcache->objects[--pcache->nobjs]);
108
+
}
109
+
110
+
static void *stage2_cache_alloc(struct kvm_mmu_page_cache *pcache)
111
+
{
112
+
void *p;
113
+
114
+
if (!pcache)
115
+
return NULL;
116
+
117
+
BUG_ON(!pcache->nobjs);
118
+
p = pcache->objects[--pcache->nobjs];
119
+
120
+
return p;
121
+
}
122
+
123
+
static bool stage2_get_leaf_entry(struct kvm *kvm, gpa_t addr,
124
+
pte_t **ptepp, u32 *ptep_level)
125
+
{
126
+
pte_t *ptep;
127
+
u32 current_level = stage2_pgd_levels - 1;
128
+
129
+
*ptep_level = current_level;
130
+
ptep = (pte_t *)kvm->arch.pgd;
131
+
ptep = &ptep[stage2_pte_index(addr, current_level)];
132
+
while (ptep && pte_val(*ptep)) {
133
+
if (stage2_pte_leaf(ptep)) {
134
+
*ptep_level = current_level;
135
+
*ptepp = ptep;
136
+
return true;
137
+
}
138
+
139
+
if (current_level) {
140
+
current_level--;
141
+
*ptep_level = current_level;
142
+
ptep = (pte_t *)stage2_pte_page_vaddr(*ptep);
143
+
ptep = &ptep[stage2_pte_index(addr, current_level)];
144
+
} else {
145
+
ptep = NULL;
146
+
}
147
+
}
148
+
149
+
return false;
150
+
}
151
+
152
+
static void stage2_remote_tlb_flush(struct kvm *kvm, u32 level, gpa_t addr)
153
+
{
154
+
struct cpumask hmask;
155
+
unsigned long size = PAGE_SIZE;
156
+
struct kvm_vmid *vmid = &kvm->arch.vmid;
157
+
158
+
if (stage2_level_to_page_size(level, &size))
159
+
return;
160
+
addr &= ~(size - 1);
161
+
162
+
/*
163
+
* TODO: Instead of cpu_online_mask, we should only target CPUs
164
+
* where the Guest/VM is running.
165
+
*/
166
+
preempt_disable();
167
+
riscv_cpuid_to_hartid_mask(cpu_online_mask, &hmask);
168
+
sbi_remote_hfence_gvma_vmid(cpumask_bits(&hmask), addr, size,
169
+
READ_ONCE(vmid->vmid));
170
+
preempt_enable();
171
+
}
172
+
173
+
static int stage2_set_pte(struct kvm *kvm, u32 level,
174
+
struct kvm_mmu_page_cache *pcache,
175
+
gpa_t addr, const pte_t *new_pte)
176
+
{
177
+
u32 current_level = stage2_pgd_levels - 1;
178
+
pte_t *next_ptep = (pte_t *)kvm->arch.pgd;
179
+
pte_t *ptep = &next_ptep[stage2_pte_index(addr, current_level)];
180
+
181
+
if (current_level < level)
182
+
return -EINVAL;
183
+
184
+
while (current_level != level) {
185
+
if (stage2_pte_leaf(ptep))
186
+
return -EEXIST;
187
+
188
+
if (!pte_val(*ptep)) {
189
+
next_ptep = stage2_cache_alloc(pcache);
190
+
if (!next_ptep)
191
+
return -ENOMEM;
192
+
*ptep = pfn_pte(PFN_DOWN(__pa(next_ptep)),
193
+
__pgprot(_PAGE_TABLE));
194
+
} else {
195
+
if (stage2_pte_leaf(ptep))
196
+
return -EEXIST;
197
+
next_ptep = (pte_t *)stage2_pte_page_vaddr(*ptep);
198
+
}
199
+
200
+
current_level--;
201
+
ptep = &next_ptep[stage2_pte_index(addr, current_level)];
202
+
}
203
+
204
+
*ptep = *new_pte;
205
+
if (stage2_pte_leaf(ptep))
206
+
stage2_remote_tlb_flush(kvm, current_level, addr);
207
+
208
+
return 0;
209
+
}
210
+
211
+
static int stage2_map_page(struct kvm *kvm,
212
+
struct kvm_mmu_page_cache *pcache,
213
+
gpa_t gpa, phys_addr_t hpa,
214
+
unsigned long page_size,
215
+
bool page_rdonly, bool page_exec)
216
+
{
217
+
int ret;
218
+
u32 level = 0;
219
+
pte_t new_pte;
220
+
pgprot_t prot;
221
+
222
+
ret = stage2_page_size_to_level(page_size, &level);
223
+
if (ret)
224
+
return ret;
225
+
226
+
/*
227
+
* A RISC-V implementation can choose to either:
228
+
* 1) Update 'A' and 'D' PTE bits in hardware
229
+
* 2) Generate page fault when 'A' and/or 'D' bits are not set
230
+
* PTE so that software can update these bits.
231
+
*
232
+
* We support both options mentioned above. To achieve this, we
233
+
* always set 'A' and 'D' PTE bits at time of creating stage2
234
+
* mapping. To support KVM dirty page logging with both options
235
+
* mentioned above, we will write-protect stage2 PTEs to track
236
+
* dirty pages.
237
+
*/
238
+
239
+
if (page_exec) {
240
+
if (page_rdonly)
241
+
prot = PAGE_READ_EXEC;
242
+
else
243
+
prot = PAGE_WRITE_EXEC;
244
+
} else {
245
+
if (page_rdonly)
246
+
prot = PAGE_READ;
247
+
else
248
+
prot = PAGE_WRITE;
249
+
}
250
+
new_pte = pfn_pte(PFN_DOWN(hpa), prot);
251
+
new_pte = pte_mkdirty(new_pte);
252
+
253
+
return stage2_set_pte(kvm, level, pcache, gpa, &new_pte);
254
+
}
255
+
256
+
enum stage2_op {
257
+
STAGE2_OP_NOP = 0, /* Nothing */
258
+
STAGE2_OP_CLEAR, /* Clear/Unmap */
259
+
STAGE2_OP_WP, /* Write-protect */
260
+
};
261
+
262
+
static void stage2_op_pte(struct kvm *kvm, gpa_t addr,
263
+
pte_t *ptep, u32 ptep_level, enum stage2_op op)
264
+
{
265
+
int i, ret;
266
+
pte_t *next_ptep;
267
+
u32 next_ptep_level;
268
+
unsigned long next_page_size, page_size;
269
+
270
+
ret = stage2_level_to_page_size(ptep_level, &page_size);
271
+
if (ret)
272
+
return;
273
+
274
+
BUG_ON(addr & (page_size - 1));
275
+
276
+
if (!pte_val(*ptep))
277
+
return;
278
+
279
+
if (ptep_level && !stage2_pte_leaf(ptep)) {
280
+
next_ptep = (pte_t *)stage2_pte_page_vaddr(*ptep);
281
+
next_ptep_level = ptep_level - 1;
282
+
ret = stage2_level_to_page_size(next_ptep_level,
283
+
&next_page_size);
284
+
if (ret)
285
+
return;
286
+
287
+
if (op == STAGE2_OP_CLEAR)
288
+
set_pte(ptep, __pte(0));
289
+
for (i = 0; i < PTRS_PER_PTE; i++)
290
+
stage2_op_pte(kvm, addr + i * next_page_size,
291
+
&next_ptep[i], next_ptep_level, op);
292
+
if (op == STAGE2_OP_CLEAR)
293
+
put_page(virt_to_page(next_ptep));
294
+
} else {
295
+
if (op == STAGE2_OP_CLEAR)
296
+
set_pte(ptep, __pte(0));
297
+
else if (op == STAGE2_OP_WP)
298
+
set_pte(ptep, __pte(pte_val(*ptep) & ~_PAGE_WRITE));
299
+
stage2_remote_tlb_flush(kvm, ptep_level, addr);
300
+
}
301
+
}
302
+
303
+
static void stage2_unmap_range(struct kvm *kvm, gpa_t start, gpa_t size)
304
+
{
305
+
int ret;
306
+
pte_t *ptep;
307
+
u32 ptep_level;
308
+
bool found_leaf;
309
+
unsigned long page_size;
310
+
gpa_t addr = start, end = start + size;
311
+
312
+
while (addr < end) {
313
+
found_leaf = stage2_get_leaf_entry(kvm, addr,
314
+
&ptep, &ptep_level);
315
+
ret = stage2_level_to_page_size(ptep_level, &page_size);
316
+
if (ret)
317
+
break;
318
+
319
+
if (!found_leaf)
320
+
goto next;
321
+
322
+
if (!(addr & (page_size - 1)) && ((end - addr) >= page_size))
323
+
stage2_op_pte(kvm, addr, ptep,
324
+
ptep_level, STAGE2_OP_CLEAR);
325
+
326
+
next:
327
+
addr += page_size;
328
+
}
329
+
}
330
+
331
+
static void stage2_wp_range(struct kvm *kvm, gpa_t start, gpa_t end)
332
+
{
333
+
int ret;
334
+
pte_t *ptep;
335
+
u32 ptep_level;
336
+
bool found_leaf;
337
+
gpa_t addr = start;
338
+
unsigned long page_size;
339
+
340
+
while (addr < end) {
341
+
found_leaf = stage2_get_leaf_entry(kvm, addr,
342
+
&ptep, &ptep_level);
343
+
ret = stage2_level_to_page_size(ptep_level, &page_size);
344
+
if (ret)
345
+
break;
346
+
347
+
if (!found_leaf)
348
+
goto next;
349
+
350
+
if (!(addr & (page_size - 1)) && ((end - addr) >= page_size))
351
+
stage2_op_pte(kvm, addr, ptep,
352
+
ptep_level, STAGE2_OP_WP);
353
+
354
+
next:
355
+
addr += page_size;
356
+
}
357
+
}
358
+
359
+
static void stage2_wp_memory_region(struct kvm *kvm, int slot)
360
+
{
361
+
struct kvm_memslots *slots = kvm_memslots(kvm);
362
+
struct kvm_memory_slot *memslot = id_to_memslot(slots, slot);
363
+
phys_addr_t start = memslot->base_gfn << PAGE_SHIFT;
364
+
phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT;
365
+
366
+
spin_lock(&kvm->mmu_lock);
367
+
stage2_wp_range(kvm, start, end);
368
+
spin_unlock(&kvm->mmu_lock);
369
+
kvm_flush_remote_tlbs(kvm);
370
+
}
371
+
372
+
static int stage2_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa,
373
+
unsigned long size, bool writable)
374
+
{
375
+
pte_t pte;
376
+
int ret = 0;
377
+
unsigned long pfn;
378
+
phys_addr_t addr, end;
379
+
struct kvm_mmu_page_cache pcache = { 0, };
380
+
381
+
end = (gpa + size + PAGE_SIZE - 1) & PAGE_MASK;
382
+
pfn = __phys_to_pfn(hpa);
383
+
384
+
for (addr = gpa; addr < end; addr += PAGE_SIZE) {
385
+
pte = pfn_pte(pfn, PAGE_KERNEL);
386
+
387
+
if (!writable)
388
+
pte = pte_wrprotect(pte);
389
+
390
+
ret = stage2_cache_topup(&pcache,
391
+
stage2_pgd_levels,
392
+
KVM_MMU_PAGE_CACHE_NR_OBJS);
393
+
if (ret)
394
+
goto out;
395
+
396
+
spin_lock(&kvm->mmu_lock);
397
+
ret = stage2_set_pte(kvm, 0, &pcache, addr, &pte);
398
+
spin_unlock(&kvm->mmu_lock);
399
+
if (ret)
400
+
goto out;
401
+
402
+
pfn++;
403
+
}
404
+
405
+
out:
406
+
stage2_cache_flush(&pcache);
407
+
return ret;
408
+
409
+
}
410
+
411
+
void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
412
+
struct kvm_memory_slot *slot,
413
+
gfn_t gfn_offset,
414
+
unsigned long mask)
415
+
{
416
+
phys_addr_t base_gfn = slot->base_gfn + gfn_offset;
417
+
phys_addr_t start = (base_gfn + __ffs(mask)) << PAGE_SHIFT;
418
+
phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT;
419
+
420
+
stage2_wp_range(kvm, start, end);
421
+
}
20
422
21
423
void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
22
424
{
425
+
}
426
+
427
+
void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm,
428
+
const struct kvm_memory_slot *memslot)
429
+
{
430
+
kvm_flush_remote_tlbs(kvm);
23
431
}
24
432
25
433
void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free)
···
440
32
441
33
void kvm_arch_flush_shadow_all(struct kvm *kvm)
442
34
{
443
-
/* TODO: */
35
+
kvm_riscv_stage2_free_pgd(kvm);
444
36
}
445
37
446
38
void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
···
454
46
const struct kvm_memory_slot *new,
455
47
enum kvm_mr_change change)
456
48
{
457
-
/* TODO: */
49
+
/*
50
+
* At this point memslot has been committed and there is an
51
+
* allocated dirty_bitmap[], dirty pages will be tracked while
52
+
* the memory slot is write protected.
53
+
*/
54
+
if (change != KVM_MR_DELETE && mem->flags & KVM_MEM_LOG_DIRTY_PAGES)
55
+
stage2_wp_memory_region(kvm, mem->slot);
458
56
}
459
57
460
58
int kvm_arch_prepare_memory_region(struct kvm *kvm,
···
468
54
const struct kvm_userspace_memory_region *mem,
469
55
enum kvm_mr_change change)
470
56
{
471
-
/* TODO: */
472
-
return 0;
57
+
hva_t hva = mem->userspace_addr;
58
+
hva_t reg_end = hva + mem->memory_size;
59
+
bool writable = !(mem->flags & KVM_MEM_READONLY);
60
+
int ret = 0;
61
+
62
+
if (change != KVM_MR_CREATE && change != KVM_MR_MOVE &&
63
+
change != KVM_MR_FLAGS_ONLY)
64
+
return 0;
65
+
66
+
/*
67
+
* Prevent userspace from creating a memory region outside of the GPA
68
+
* space addressable by the KVM guest GPA space.
69
+
*/
70
+
if ((memslot->base_gfn + memslot->npages) >=
71
+
(stage2_gpa_size >> PAGE_SHIFT))
72
+
return -EFAULT;
73
+
74
+
mmap_read_lock(current->mm);
75
+
76
+
/*
77
+
* A memory region could potentially cover multiple VMAs, and
78
+
* any holes between them, so iterate over all of them to find
79
+
* out if we can map any of them right now.
80
+
*
81
+
* +--------------------------------------------+
82
+
* +---------------+----------------+ +----------------+
83
+
* | : VMA 1 | VMA 2 | | VMA 3 : |
84
+
* +---------------+----------------+ +----------------+
85
+
* | memory region |
86
+
* +--------------------------------------------+
87
+
*/
88
+
do {
89
+
struct vm_area_struct *vma = find_vma(current->mm, hva);
90
+
hva_t vm_start, vm_end;
91
+
92
+
if (!vma || vma->vm_start >= reg_end)
93
+
break;
94
+
95
+
/*
96
+
* Mapping a read-only VMA is only allowed if the
97
+
* memory region is configured as read-only.
98
+
*/
99
+
if (writable && !(vma->vm_flags & VM_WRITE)) {
100
+
ret = -EPERM;
101
+
break;
102
+
}
103
+
104
+
/* Take the intersection of this VMA with the memory region */
105
+
vm_start = max(hva, vma->vm_start);
106
+
vm_end = min(reg_end, vma->vm_end);
107
+
108
+
if (vma->vm_flags & VM_PFNMAP) {
109
+
gpa_t gpa = mem->guest_phys_addr +
110
+
(vm_start - mem->userspace_addr);
111
+
phys_addr_t pa;
112
+
113
+
pa = (phys_addr_t)vma->vm_pgoff << PAGE_SHIFT;
114
+
pa += vm_start - vma->vm_start;
115
+
116
+
/* IO region dirty page logging not allowed */
117
+
if (memslot->flags & KVM_MEM_LOG_DIRTY_PAGES) {
118
+
ret = -EINVAL;
119
+
goto out;
120
+
}
121
+
122
+
ret = stage2_ioremap(kvm, gpa, pa,
123
+
vm_end - vm_start, writable);
124
+
if (ret)
125
+
break;
126
+
}
127
+
hva = vm_end;
128
+
} while (hva < reg_end);
129
+
130
+
if (change == KVM_MR_FLAGS_ONLY)
131
+
goto out;
132
+
133
+
spin_lock(&kvm->mmu_lock);
134
+
if (ret)
135
+
stage2_unmap_range(kvm, mem->guest_phys_addr,
136
+
mem->memory_size);
137
+
spin_unlock(&kvm->mmu_lock);
138
+
139
+
out:
140
+
mmap_read_unlock(current->mm);
141
+
return ret;
473
142
}
474
143
475
144
int kvm_riscv_stage2_map(struct kvm_vcpu *vcpu,
476
145
struct kvm_memory_slot *memslot,
477
146
gpa_t gpa, unsigned long hva, bool is_write)
478
147
{
479
-
/* TODO: */
480
-
return 0;
148
+
int ret;
149
+
kvm_pfn_t hfn;
150
+
bool writeable;
151
+
short vma_pageshift;
152
+
gfn_t gfn = gpa >> PAGE_SHIFT;
153
+
struct vm_area_struct *vma;
154
+
struct kvm *kvm = vcpu->kvm;
155
+
struct kvm_mmu_page_cache *pcache = &vcpu->arch.mmu_page_cache;
156
+
bool logging = (memslot->dirty_bitmap &&
157
+
!(memslot->flags & KVM_MEM_READONLY)) ? true : false;
158
+
unsigned long vma_pagesize;
159
+
160
+
mmap_read_lock(current->mm);
161
+
162
+
vma = find_vma_intersection(current->mm, hva, hva + 1);
163
+
if (unlikely(!vma)) {
164
+
kvm_err("Failed to find VMA for hva 0x%lx\n", hva);
165
+
mmap_read_unlock(current->mm);
166
+
return -EFAULT;
167
+
}
168
+
169
+
if (is_vm_hugetlb_page(vma))
170
+
vma_pageshift = huge_page_shift(hstate_vma(vma));
171
+
else
172
+
vma_pageshift = PAGE_SHIFT;
173
+
vma_pagesize = 1ULL << vma_pageshift;
174
+
if (logging || (vma->vm_flags & VM_PFNMAP))
175
+
vma_pagesize = PAGE_SIZE;
176
+
177
+
if (vma_pagesize == PMD_SIZE || vma_pagesize == PGDIR_SIZE)
178
+
gfn = (gpa & huge_page_mask(hstate_vma(vma))) >> PAGE_SHIFT;
179
+
180
+
mmap_read_unlock(current->mm);
181
+
182
+
if (vma_pagesize != PGDIR_SIZE &&
183
+
vma_pagesize != PMD_SIZE &&
184
+
vma_pagesize != PAGE_SIZE) {
185
+
kvm_err("Invalid VMA page size 0x%lx\n", vma_pagesize);
186
+
return -EFAULT;
187
+
}
188
+
189
+
/* We need minimum second+third level pages */
190
+
ret = stage2_cache_topup(pcache, stage2_pgd_levels,
191
+
KVM_MMU_PAGE_CACHE_NR_OBJS);
192
+
if (ret) {
193
+
kvm_err("Failed to topup stage2 cache\n");
194
+
return ret;
195
+
}
196
+
197
+
hfn = gfn_to_pfn_prot(kvm, gfn, is_write, &writeable);
198
+
if (hfn == KVM_PFN_ERR_HWPOISON) {
199
+
send_sig_mceerr(BUS_MCEERR_AR, (void __user *)hva,
200
+
vma_pageshift, current);
201
+
return 0;
202
+
}
203
+
if (is_error_noslot_pfn(hfn))
204
+
return -EFAULT;
205
+
206
+
/*
207
+
* If logging is active then we allow writable pages only
208
+
* for write faults.
209
+
*/
210
+
if (logging && !is_write)
211
+
writeable = false;
212
+
213
+
spin_lock(&kvm->mmu_lock);
214
+
215
+
if (writeable) {
216
+
kvm_set_pfn_dirty(hfn);
217
+
mark_page_dirty(kvm, gfn);
218
+
ret = stage2_map_page(kvm, pcache, gpa, hfn << PAGE_SHIFT,
219
+
vma_pagesize, false, true);
220
+
} else {
221
+
ret = stage2_map_page(kvm, pcache, gpa, hfn << PAGE_SHIFT,
222
+
vma_pagesize, true, true);
223
+
}
224
+
225
+
if (ret)
226
+
kvm_err("Failed to map in stage2\n");
227
+
228
+
spin_unlock(&kvm->mmu_lock);
229
+
kvm_set_pfn_accessed(hfn);
230
+
kvm_release_pfn_clean(hfn);
231
+
return ret;
481
232
}
482
233
483
234
void kvm_riscv_stage2_flush_cache(struct kvm_vcpu *vcpu)
484
235
{
485
-
/* TODO: */
236
+
stage2_cache_flush(&vcpu->arch.mmu_page_cache);
486
237
}
487
238
488
239
int kvm_riscv_stage2_alloc_pgd(struct kvm *kvm)
489
240
{
490
-
/* TODO: */
241
+
struct page *pgd_page;
242
+
243
+
if (kvm->arch.pgd != NULL) {
244
+
kvm_err("kvm_arch already initialized?\n");
245
+
return -EINVAL;
246
+
}
247
+
248
+
pgd_page = alloc_pages(GFP_KERNEL | __GFP_ZERO,
249
+
get_order(stage2_pgd_size));
250
+
if (!pgd_page)
251
+
return -ENOMEM;
252
+
kvm->arch.pgd = page_to_virt(pgd_page);
253
+
kvm->arch.pgd_phys = page_to_phys(pgd_page);
254
+
491
255
return 0;
492
256
}
493
257
494
258
void kvm_riscv_stage2_free_pgd(struct kvm *kvm)
495
259
{
496
-
/* TODO: */
260
+
void *pgd = NULL;
261
+
262
+
spin_lock(&kvm->mmu_lock);
263
+
if (kvm->arch.pgd) {
264
+
stage2_unmap_range(kvm, 0UL, stage2_gpa_size);
265
+
pgd = READ_ONCE(kvm->arch.pgd);
266
+
kvm->arch.pgd = NULL;
267
+
kvm->arch.pgd_phys = 0;
268
+
}
269
+
spin_unlock(&kvm->mmu_lock);
270
+
271
+
if (pgd)
272
+
free_pages((unsigned long)pgd, get_order(stage2_pgd_size));
497
273
}
498
274
499
275
void kvm_riscv_stage2_update_hgatp(struct kvm_vcpu *vcpu)
500
276
{
501
-
/* TODO: */
277
+
unsigned long hgatp = stage2_mode;
278
+
struct kvm_arch *k = &vcpu->kvm->arch;
279
+
280
+
hgatp |= (READ_ONCE(k->vmid.vmid) << HGATP_VMID_SHIFT) &
281
+
HGATP_VMID_MASK;
282
+
hgatp |= (k->pgd_phys >> PAGE_SHIFT) & HGATP_PPN;
283
+
284
+
csr_write(CSR_HGATP, hgatp);
285
+
286
+
if (!kvm_riscv_stage2_vmid_bits())
287
+
__kvm_riscv_hfence_gvma_all();
288
+
}
289
+
290
+
void kvm_riscv_stage2_mode_detect(void)
291
+
{
292
+
#ifdef CONFIG_64BIT
293
+
/* Try Sv48x4 stage2 mode */
294
+
csr_write(CSR_HGATP, HGATP_MODE_SV48X4 << HGATP_MODE_SHIFT);
295
+
if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV48X4) {
296
+
stage2_mode = (HGATP_MODE_SV48X4 << HGATP_MODE_SHIFT);
297
+
stage2_pgd_levels = 4;
298
+
}
299
+
csr_write(CSR_HGATP, 0);
300
+
301
+
__kvm_riscv_hfence_gvma_all();
302
+
#endif
303
+
}
304
+
305
+
unsigned long kvm_riscv_stage2_mode(void)
306
+
{
307
+
return stage2_mode >> HGATP_MODE_SHIFT;
502
308
}
-6
arch/riscv/kvm/vm.c
-6
arch/riscv/kvm/vm.c