Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
KVM: S390: Remove PGSTE code from linux/s390 mm
Remove the PGSTE config option.
Remove all code from linux/s390 mm that involves PGSTEs.
Acked-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Claudio Imbrenda <imbrenda@linux.ibm.com>
+15
-1021
-3
arch/s390/Kconfig
-3
arch/s390/Kconfig
-6
arch/s390/include/asm/hugetlb.h
-6
arch/s390/include/asm/hugetlb.h
···
37
37
return __huge_ptep_get_and_clear(mm, addr, ptep);
38
38
}
39
39
40
-
static inline void arch_clear_hugetlb_flags(struct folio *folio)
41
-
{
42
-
clear_bit(PG_arch_1, &folio->flags.f);
43
-
}
44
-
#define arch_clear_hugetlb_flags arch_clear_hugetlb_flags
45
-
46
40
#define __HAVE_ARCH_HUGE_PTE_CLEAR
47
41
static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
48
42
pte_t *ptep, unsigned long sz)
-13
arch/s390/include/asm/mmu.h
-13
arch/s390/include/asm/mmu.h
···
19
19
/* The mmu context belongs to a secure guest. */
20
20
atomic_t protected_count;
21
21
/*
22
-
* The following bitfields need a down_write on the mm
23
-
* semaphore when they are written to. As they are only
24
-
* written once, they can be read without a lock.
25
-
*/
26
-
/* The mmu context uses extended page tables. */
27
-
unsigned int has_pgste:1;
28
-
/* The mmu context uses storage keys. */
29
-
unsigned int uses_skeys:1;
30
-
/* The mmu context uses CMM. */
31
-
unsigned int uses_cmm:1;
32
-
/*
33
22
* The mmu context allows COW-sharing of memory pages (KSM, zeropage).
34
23
* Note that COW-sharing during fork() is currently always allowed.
35
24
*/
36
25
unsigned int allow_cow_sharing:1;
37
-
/* The gmaps associated with this context are allowed to use huge pages. */
38
-
unsigned int allow_gmap_hpage_1m:1;
39
26
} mm_context_t;
40
27
41
28
#define INIT_MM_CONTEXT(name) \
-4
arch/s390/include/asm/page.h
-4
arch/s390/include/asm/page.h
···
78
78
#ifdef STRICT_MM_TYPECHECKS
79
79
80
80
typedef struct { unsigned long pgprot; } pgprot_t;
81
-
typedef struct { unsigned long pgste; } pgste_t;
82
81
typedef struct { unsigned long pte; } pte_t;
83
82
typedef struct { unsigned long pmd; } pmd_t;
84
83
typedef struct { unsigned long pud; } pud_t;
···
93
94
#else /* STRICT_MM_TYPECHECKS */
94
95
95
96
typedef unsigned long pgprot_t;
96
-
typedef unsigned long pgste_t;
97
97
typedef unsigned long pte_t;
98
98
typedef unsigned long pmd_t;
99
99
typedef unsigned long pud_t;
···
108
110
#endif /* STRICT_MM_TYPECHECKS */
109
111
110
112
DEFINE_PGVAL_FUNC(pgprot)
111
-
DEFINE_PGVAL_FUNC(pgste)
112
113
DEFINE_PGVAL_FUNC(pte)
113
114
DEFINE_PGVAL_FUNC(pmd)
114
115
DEFINE_PGVAL_FUNC(pud)
···
117
120
typedef pte_t *pgtable_t;
118
121
119
122
#define __pgprot(x) ((pgprot_t) { (x) } )
120
-
#define __pgste(x) ((pgste_t) { (x) } )
121
123
#define __pte(x) ((pte_t) { (x) } )
122
124
#define __pmd(x) ((pmd_t) { (x) } )
123
125
#define __pud(x) ((pud_t) { (x) } )
-4
arch/s390/include/asm/pgalloc.h
-4
arch/s390/include/asm/pgalloc.h
···
27
27
#define page_table_alloc(...) alloc_hooks(page_table_alloc_noprof(__VA_ARGS__))
28
28
void page_table_free(struct mm_struct *, unsigned long *);
29
29
30
-
struct ptdesc *page_table_alloc_pgste_noprof(struct mm_struct *mm);
31
-
#define page_table_alloc_pgste(...) alloc_hooks(page_table_alloc_pgste_noprof(__VA_ARGS__))
32
-
void page_table_free_pgste(struct ptdesc *ptdesc);
33
-
34
30
static inline void crst_table_init(unsigned long *crst, unsigned long entry)
35
31
{
36
32
memset64((u64 *)crst, entry, _CRST_ENTRIES);
+7
-114
arch/s390/include/asm/pgtable.h
+7
-114
arch/s390/include/asm/pgtable.h
···
413
413
* SW-bits: y young, d dirty, r read, w write
414
414
*/
415
415
416
-
/* Page status table bits for virtualization */
417
-
#define PGSTE_ACC_BITS 0xf000000000000000UL
418
-
#define PGSTE_FP_BIT 0x0800000000000000UL
419
-
#define PGSTE_PCL_BIT 0x0080000000000000UL
420
-
#define PGSTE_HR_BIT 0x0040000000000000UL
421
-
#define PGSTE_HC_BIT 0x0020000000000000UL
422
-
#define PGSTE_GR_BIT 0x0004000000000000UL
423
-
#define PGSTE_GC_BIT 0x0002000000000000UL
424
-
#define PGSTE_ST2_MASK 0x0000ffff00000000UL
425
-
#define PGSTE_UC_BIT 0x0000000000008000UL /* user dirty (migration) */
426
-
#define PGSTE_IN_BIT 0x0000000000004000UL /* IPTE notify bit */
427
-
#define PGSTE_VSIE_BIT 0x0000000000002000UL /* ref'd in a shadow table */
428
-
429
-
/* Guest Page State used for virtualization */
430
-
#define _PGSTE_GPS_ZERO 0x0000000080000000UL
431
-
#define _PGSTE_GPS_NODAT 0x0000000040000000UL
432
-
#define _PGSTE_GPS_USAGE_MASK 0x0000000003000000UL
433
-
#define _PGSTE_GPS_USAGE_STABLE 0x0000000000000000UL
434
-
#define _PGSTE_GPS_USAGE_UNUSED 0x0000000001000000UL
435
-
#define _PGSTE_GPS_USAGE_POT_VOLATILE 0x0000000002000000UL
436
-
#define _PGSTE_GPS_USAGE_VOLATILE _PGSTE_GPS_USAGE_MASK
437
-
438
416
/*
439
417
* A user page table pointer has the space-switch-event bit, the
440
418
* private-space-control bit and the storage-alteration-event-control
···
544
566
}
545
567
#define mm_pmd_folded(mm) mm_pmd_folded(mm)
546
568
547
-
static inline int mm_has_pgste(struct mm_struct *mm)
548
-
{
549
-
#ifdef CONFIG_PGSTE
550
-
if (unlikely(mm->context.has_pgste))
551
-
return 1;
552
-
#endif
553
-
return 0;
554
-
}
555
-
556
569
static inline int mm_is_protected(struct mm_struct *mm)
557
570
{
558
571
#if IS_ENABLED(CONFIG_KVM)
···
551
582
return 1;
552
583
#endif
553
584
return 0;
554
-
}
555
-
556
-
static inline pgste_t clear_pgste_bit(pgste_t pgste, unsigned long mask)
557
-
{
558
-
return __pgste(pgste_val(pgste) & ~mask);
559
-
}
560
-
561
-
static inline pgste_t set_pgste_bit(pgste_t pgste, unsigned long mask)
562
-
{
563
-
return __pgste(pgste_val(pgste) | mask);
564
585
}
565
586
566
587
static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot)
···
593
634
{
594
635
#if IS_ENABLED(CONFIG_KVM)
595
636
if (!mm->context.allow_cow_sharing)
596
-
return 1;
597
-
#endif
598
-
return 0;
599
-
}
600
-
601
-
static inline int mm_uses_skeys(struct mm_struct *mm)
602
-
{
603
-
#ifdef CONFIG_PGSTE
604
-
if (mm->context.uses_skeys)
605
637
return 1;
606
638
#endif
607
639
return 0;
···
1306
1356
{
1307
1357
if (pte_same(*ptep, entry))
1308
1358
return 0;
1309
-
if (cpu_has_rdp() && !mm_has_pgste(vma->vm_mm) && pte_allow_rdp(*ptep, entry))
1359
+
if (cpu_has_rdp() && pte_allow_rdp(*ptep, entry))
1310
1360
ptep_reset_dat_prot(vma->vm_mm, addr, ptep, entry);
1311
1361
else
1312
1362
ptep_xchg_direct(vma->vm_mm, addr, ptep, entry);
1313
1363
return 1;
1314
1364
}
1315
-
1316
-
/*
1317
-
* Additional functions to handle KVM guest page tables
1318
-
*/
1319
-
void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr,
1320
-
pte_t *ptep, pte_t entry);
1321
-
void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
1322
-
int ptep_force_prot(struct mm_struct *mm, unsigned long gaddr,
1323
-
pte_t *ptep, int prot, unsigned long bit);
1324
-
void ptep_zap_unused(struct mm_struct *mm, unsigned long addr,
1325
-
pte_t *ptep , int reset);
1326
-
void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
1327
-
int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr,
1328
-
pte_t *sptep, pte_t *tptep, pte_t pte);
1329
-
void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep);
1330
-
1331
-
bool ptep_test_and_clear_uc(struct mm_struct *mm, unsigned long address,
1332
-
pte_t *ptep);
1333
-
int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
1334
-
unsigned char key, bool nq);
1335
-
int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
1336
-
unsigned char key, unsigned char *oldkey,
1337
-
bool nq, bool mr, bool mc);
1338
-
int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr);
1339
-
int get_guest_storage_key(struct mm_struct *mm, unsigned long addr,
1340
-
unsigned char *key);
1341
-
1342
-
int set_pgste_bits(struct mm_struct *mm, unsigned long addr,
1343
-
unsigned long bits, unsigned long value);
1344
-
int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep);
1345
-
int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc,
1346
-
unsigned long *oldpte, unsigned long *oldpgste);
1347
1365
1348
1366
#define pgprot_writecombine pgprot_writecombine
1349
1367
pgprot_t pgprot_writecombine(pgprot_t prot);
···
1327
1409
{
1328
1410
if (pte_present(entry))
1329
1411
entry = clear_pte_bit(entry, __pgprot(_PAGE_UNUSED));
1330
-
if (mm_has_pgste(mm)) {
1331
-
for (;;) {
1332
-
ptep_set_pte_at(mm, addr, ptep, entry);
1333
-
if (--nr == 0)
1334
-
break;
1335
-
ptep++;
1336
-
entry = __pte(pte_val(entry) + PAGE_SIZE);
1337
-
addr += PAGE_SIZE;
1338
-
}
1339
-
} else {
1340
-
for (;;) {
1341
-
set_pte(ptep, entry);
1342
-
if (--nr == 0)
1343
-
break;
1344
-
ptep++;
1345
-
entry = __pte(pte_val(entry) + PAGE_SIZE);
1346
-
}
1412
+
for (;;) {
1413
+
set_pte(ptep, entry);
1414
+
if (--nr == 0)
1415
+
break;
1416
+
ptep++;
1417
+
entry = __pte(pte_val(entry) + PAGE_SIZE);
1347
1418
}
1348
1419
}
1349
1420
#define set_ptes set_ptes
···
1932
2025
1933
2026
#define pmd_pgtable(pmd) \
1934
2027
((pgtable_t)__va(pmd_val(pmd) & -sizeof(pte_t)*PTRS_PER_PTE))
1935
-
1936
-
static inline unsigned long gmap_pgste_get_pgt_addr(unsigned long *pgt)
1937
-
{
1938
-
unsigned long *pgstes, res;
1939
-
1940
-
pgstes = pgt + _PAGE_ENTRIES;
1941
-
1942
-
res = (pgstes[0] & PGSTE_ST2_MASK) << 16;
1943
-
res |= pgstes[1] & PGSTE_ST2_MASK;
1944
-
res |= (pgstes[2] & PGSTE_ST2_MASK) >> 16;
1945
-
res |= (pgstes[3] & PGSTE_ST2_MASK) >> 32;
1946
-
1947
-
return res;
1948
-
}
1949
2028
1950
2029
#endif /* _S390_PAGE_H */
+1
arch/s390/kvm/dat.h
+1
arch/s390/kvm/dat.h
-24
arch/s390/mm/hugetlbpage.c
-24
arch/s390/mm/hugetlbpage.c
···
135
135
return __pte(pteval);
136
136
}
137
137
138
-
static void clear_huge_pte_skeys(struct mm_struct *mm, unsigned long rste)
139
-
{
140
-
struct folio *folio;
141
-
unsigned long size, paddr;
142
-
143
-
if (!mm_uses_skeys(mm) ||
144
-
rste & _SEGMENT_ENTRY_INVALID)
145
-
return;
146
-
147
-
if ((rste & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) {
148
-
folio = page_folio(pud_page(__pud(rste)));
149
-
size = PUD_SIZE;
150
-
paddr = rste & PUD_MASK;
151
-
} else {
152
-
folio = page_folio(pmd_page(__pmd(rste)));
153
-
size = PMD_SIZE;
154
-
paddr = rste & PMD_MASK;
155
-
}
156
-
157
-
if (!test_and_set_bit(PG_arch_1, &folio->flags.f))
158
-
__storage_key_init_range(paddr, paddr + size);
159
-
}
160
-
161
138
void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
162
139
pte_t *ptep, pte_t pte)
163
140
{
···
150
173
} else if (likely(pte_present(pte)))
151
174
rste |= _SEGMENT_ENTRY_LARGE;
152
175
153
-
clear_huge_pte_skeys(mm, rste);
154
176
set_pte(ptep, __pte(rste));
155
177
}
156
178
-24
arch/s390/mm/pgalloc.c
-24
arch/s390/mm/pgalloc.c
···
114
114
return -ENOMEM;
115
115
}
116
116
117
-
#ifdef CONFIG_PGSTE
118
-
119
-
struct ptdesc *page_table_alloc_pgste_noprof(struct mm_struct *mm)
120
-
{
121
-
struct ptdesc *ptdesc;
122
-
u64 *table;
123
-
124
-
ptdesc = pagetable_alloc_noprof(GFP_KERNEL_ACCOUNT, 0);
125
-
if (ptdesc) {
126
-
table = (u64 *)ptdesc_address(ptdesc);
127
-
__arch_set_page_dat(table, 1);
128
-
memset64(table, _PAGE_INVALID, PTRS_PER_PTE);
129
-
memset64(table + PTRS_PER_PTE, 0, PTRS_PER_PTE);
130
-
}
131
-
return ptdesc;
132
-
}
133
-
134
-
void page_table_free_pgste(struct ptdesc *ptdesc)
135
-
{
136
-
pagetable_free(ptdesc);
137
-
}
138
-
139
-
#endif /* CONFIG_PGSTE */
140
-
141
117
unsigned long *page_table_alloc_noprof(struct mm_struct *mm)
142
118
{
143
119
gfp_t gfp = GFP_KERNEL_ACCOUNT;
+7
-820
arch/s390/mm/pgtable.c
+7
-820
arch/s390/mm/pgtable.c
···
115
115
return old;
116
116
}
117
117
118
-
static inline pgste_t pgste_get_lock(pte_t *ptep)
119
-
{
120
-
unsigned long value = 0;
121
-
#ifdef CONFIG_PGSTE
122
-
unsigned long *ptr = (unsigned long *)(ptep + PTRS_PER_PTE);
123
-
124
-
do {
125
-
value = __atomic64_or_barrier(PGSTE_PCL_BIT, ptr);
126
-
} while (value & PGSTE_PCL_BIT);
127
-
value |= PGSTE_PCL_BIT;
128
-
#endif
129
-
return __pgste(value);
130
-
}
131
-
132
-
static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste)
133
-
{
134
-
#ifdef CONFIG_PGSTE
135
-
barrier();
136
-
WRITE_ONCE(*(unsigned long *)(ptep + PTRS_PER_PTE), pgste_val(pgste) & ~PGSTE_PCL_BIT);
137
-
#endif
138
-
}
139
-
140
-
static inline pgste_t pgste_get(pte_t *ptep)
141
-
{
142
-
unsigned long pgste = 0;
143
-
#ifdef CONFIG_PGSTE
144
-
pgste = *(unsigned long *)(ptep + PTRS_PER_PTE);
145
-
#endif
146
-
return __pgste(pgste);
147
-
}
148
-
149
-
static inline void pgste_set(pte_t *ptep, pgste_t pgste)
150
-
{
151
-
#ifdef CONFIG_PGSTE
152
-
*(pgste_t *)(ptep + PTRS_PER_PTE) = pgste;
153
-
#endif
154
-
}
155
-
156
-
static inline pgste_t pgste_update_all(pte_t pte, pgste_t pgste,
157
-
struct mm_struct *mm)
158
-
{
159
-
#ifdef CONFIG_PGSTE
160
-
unsigned long address, bits, skey;
161
-
162
-
if (!mm_uses_skeys(mm) || pte_val(pte) & _PAGE_INVALID)
163
-
return pgste;
164
-
address = pte_val(pte) & PAGE_MASK;
165
-
skey = (unsigned long) page_get_storage_key(address);
166
-
bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
167
-
/* Transfer page changed & referenced bit to guest bits in pgste */
168
-
pgste = set_pgste_bit(pgste, bits << 48); /* GR bit & GC bit */
169
-
/* Copy page access key and fetch protection bit to pgste */
170
-
pgste = clear_pgste_bit(pgste, PGSTE_ACC_BITS | PGSTE_FP_BIT);
171
-
pgste = set_pgste_bit(pgste, (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56);
172
-
#endif
173
-
return pgste;
174
-
175
-
}
176
-
177
-
static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry,
178
-
struct mm_struct *mm)
179
-
{
180
-
#ifdef CONFIG_PGSTE
181
-
unsigned long address;
182
-
unsigned long nkey;
183
-
184
-
if (!mm_uses_skeys(mm) || pte_val(entry) & _PAGE_INVALID)
185
-
return;
186
-
VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID));
187
-
address = pte_val(entry) & PAGE_MASK;
188
-
/*
189
-
* Set page access key and fetch protection bit from pgste.
190
-
* The guest C/R information is still in the PGSTE, set real
191
-
* key C/R to 0.
192
-
*/
193
-
nkey = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56;
194
-
nkey |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48;
195
-
page_set_storage_key(address, nkey, 0);
196
-
#endif
197
-
}
198
-
199
-
static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry)
200
-
{
201
-
#ifdef CONFIG_PGSTE
202
-
if ((pte_val(entry) & _PAGE_PRESENT) &&
203
-
(pte_val(entry) & _PAGE_WRITE) &&
204
-
!(pte_val(entry) & _PAGE_INVALID)) {
205
-
if (!machine_has_esop()) {
206
-
/*
207
-
* Without enhanced suppression-on-protection force
208
-
* the dirty bit on for all writable ptes.
209
-
*/
210
-
entry = set_pte_bit(entry, __pgprot(_PAGE_DIRTY));
211
-
entry = clear_pte_bit(entry, __pgprot(_PAGE_PROTECT));
212
-
}
213
-
if (!(pte_val(entry) & _PAGE_PROTECT))
214
-
/* This pte allows write access, set user-dirty */
215
-
pgste = set_pgste_bit(pgste, PGSTE_UC_BIT);
216
-
}
217
-
#endif
218
-
set_pte(ptep, entry);
219
-
return pgste;
220
-
}
221
-
222
-
static inline pgste_t pgste_pte_notify(struct mm_struct *mm,
223
-
unsigned long addr,
224
-
pte_t *ptep, pgste_t pgste)
225
-
{
226
-
#ifdef CONFIG_PGSTE
227
-
unsigned long bits;
228
-
229
-
bits = pgste_val(pgste) & (PGSTE_IN_BIT | PGSTE_VSIE_BIT);
230
-
if (bits) {
231
-
pgste = __pgste(pgste_val(pgste) ^ bits);
232
-
ptep_notify(mm, addr, ptep, bits);
233
-
}
234
-
#endif
235
-
return pgste;
236
-
}
237
-
238
-
static inline pgste_t ptep_xchg_start(struct mm_struct *mm,
239
-
unsigned long addr, pte_t *ptep)
240
-
{
241
-
pgste_t pgste = __pgste(0);
242
-
243
-
if (mm_has_pgste(mm)) {
244
-
pgste = pgste_get_lock(ptep);
245
-
pgste = pgste_pte_notify(mm, addr, ptep, pgste);
246
-
}
247
-
return pgste;
248
-
}
249
-
250
-
static inline pte_t ptep_xchg_commit(struct mm_struct *mm,
251
-
unsigned long addr, pte_t *ptep,
252
-
pgste_t pgste, pte_t old, pte_t new)
253
-
{
254
-
if (mm_has_pgste(mm)) {
255
-
if (pte_val(old) & _PAGE_INVALID)
256
-
pgste_set_key(ptep, pgste, new, mm);
257
-
if (pte_val(new) & _PAGE_INVALID) {
258
-
pgste = pgste_update_all(old, pgste, mm);
259
-
if ((pgste_val(pgste) & _PGSTE_GPS_USAGE_MASK) ==
260
-
_PGSTE_GPS_USAGE_UNUSED)
261
-
old = set_pte_bit(old, __pgprot(_PAGE_UNUSED));
262
-
}
263
-
pgste = pgste_set_pte(ptep, pgste, new);
264
-
pgste_set_unlock(ptep, pgste);
265
-
} else {
266
-
set_pte(ptep, new);
267
-
}
268
-
return old;
269
-
}
270
-
271
118
pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr,
272
119
pte_t *ptep, pte_t new)
273
120
{
274
-
pgste_t pgste;
275
121
pte_t old;
276
-
int nodat;
277
122
278
123
preempt_disable();
279
-
pgste = ptep_xchg_start(mm, addr, ptep);
280
-
nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
281
-
old = ptep_flush_direct(mm, addr, ptep, nodat);
282
-
old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
124
+
old = ptep_flush_direct(mm, addr, ptep, 1);
125
+
set_pte(ptep, new);
283
126
preempt_enable();
284
127
return old;
285
128
}
···
156
313
pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr,
157
314
pte_t *ptep, pte_t new)
158
315
{
159
-
pgste_t pgste;
160
316
pte_t old;
161
-
int nodat;
162
317
163
318
preempt_disable();
164
-
pgste = ptep_xchg_start(mm, addr, ptep);
165
-
nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
166
-
old = ptep_flush_lazy(mm, addr, ptep, nodat);
167
-
old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
319
+
old = ptep_flush_lazy(mm, addr, ptep, 1);
320
+
set_pte(ptep, new);
168
321
preempt_enable();
169
322
return old;
170
323
}
···
169
330
pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr,
170
331
pte_t *ptep)
171
332
{
172
-
pgste_t pgste;
173
-
pte_t old;
174
-
int nodat;
175
-
struct mm_struct *mm = vma->vm_mm;
176
-
177
-
pgste = ptep_xchg_start(mm, addr, ptep);
178
-
nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
179
-
old = ptep_flush_lazy(mm, addr, ptep, nodat);
180
-
if (mm_has_pgste(mm)) {
181
-
pgste = pgste_update_all(old, pgste, mm);
182
-
pgste_set(ptep, pgste);
183
-
}
184
-
return old;
333
+
return ptep_flush_lazy(vma->vm_mm, addr, ptep, 1);
185
334
}
186
335
187
336
void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr,
188
337
pte_t *ptep, pte_t old_pte, pte_t pte)
189
338
{
190
-
pgste_t pgste;
191
-
struct mm_struct *mm = vma->vm_mm;
192
-
193
-
if (mm_has_pgste(mm)) {
194
-
pgste = pgste_get(ptep);
195
-
pgste_set_key(ptep, pgste, pte, mm);
196
-
pgste = pgste_set_pte(ptep, pgste, pte);
197
-
pgste_set_unlock(ptep, pgste);
198
-
} else {
199
-
set_pte(ptep, pte);
200
-
}
339
+
set_pte(ptep, pte);
201
340
}
202
341
203
342
static inline void pmdp_idte_local(struct mm_struct *mm,
204
343
unsigned long addr, pmd_t *pmdp)
205
344
{
206
345
if (machine_has_tlb_guest())
207
-
__pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE,
208
-
mm->context.asce, IDTE_LOCAL);
346
+
__pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE, mm->context.asce, IDTE_LOCAL);
209
347
else
210
348
__pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL);
211
349
}
···
235
419
atomic_dec(&mm->context.flush_count);
236
420
return old;
237
421
}
238
-
239
-
#ifdef CONFIG_PGSTE
240
-
static int pmd_lookup(struct mm_struct *mm, unsigned long addr, pmd_t **pmdp)
241
-
{
242
-
struct vm_area_struct *vma;
243
-
pgd_t *pgd;
244
-
p4d_t *p4d;
245
-
pud_t *pud;
246
-
247
-
/* We need a valid VMA, otherwise this is clearly a fault. */
248
-
vma = vma_lookup(mm, addr);
249
-
if (!vma)
250
-
return -EFAULT;
251
-
252
-
pgd = pgd_offset(mm, addr);
253
-
if (!pgd_present(*pgd))
254
-
return -ENOENT;
255
-
256
-
p4d = p4d_offset(pgd, addr);
257
-
if (!p4d_present(*p4d))
258
-
return -ENOENT;
259
-
260
-
pud = pud_offset(p4d, addr);
261
-
if (!pud_present(*pud))
262
-
return -ENOENT;
263
-
264
-
/* Large PUDs are not supported yet. */
265
-
if (pud_leaf(*pud))
266
-
return -EFAULT;
267
-
268
-
*pmdp = pmd_offset(pud, addr);
269
-
return 0;
270
-
}
271
-
#endif
272
422
273
423
pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr,
274
424
pmd_t *pmdp, pmd_t new)
···
353
571
return pgtable;
354
572
}
355
573
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
356
-
357
-
#ifdef CONFIG_PGSTE
358
-
void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr,
359
-
pte_t *ptep, pte_t entry)
360
-
{
361
-
pgste_t pgste;
362
-
363
-
/* the mm_has_pgste() check is done in set_pte_at() */
364
-
preempt_disable();
365
-
pgste = pgste_get_lock(ptep);
366
-
pgste = clear_pgste_bit(pgste, _PGSTE_GPS_ZERO);
367
-
pgste_set_key(ptep, pgste, entry, mm);
368
-
pgste = pgste_set_pte(ptep, pgste, entry);
369
-
pgste_set_unlock(ptep, pgste);
370
-
preempt_enable();
371
-
}
372
-
373
-
void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
374
-
{
375
-
pgste_t pgste;
376
-
377
-
preempt_disable();
378
-
pgste = pgste_get_lock(ptep);
379
-
pgste = set_pgste_bit(pgste, PGSTE_IN_BIT);
380
-
pgste_set_unlock(ptep, pgste);
381
-
preempt_enable();
382
-
}
383
-
384
-
/**
385
-
* ptep_force_prot - change access rights of a locked pte
386
-
* @mm: pointer to the process mm_struct
387
-
* @addr: virtual address in the guest address space
388
-
* @ptep: pointer to the page table entry
389
-
* @prot: indicates guest access rights: PROT_NONE, PROT_READ or PROT_WRITE
390
-
* @bit: pgste bit to set (e.g. for notification)
391
-
*
392
-
* Returns 0 if the access rights were changed and -EAGAIN if the current
393
-
* and requested access rights are incompatible.
394
-
*/
395
-
int ptep_force_prot(struct mm_struct *mm, unsigned long addr,
396
-
pte_t *ptep, int prot, unsigned long bit)
397
-
{
398
-
pte_t entry;
399
-
pgste_t pgste;
400
-
int pte_i, pte_p, nodat;
401
-
402
-
pgste = pgste_get_lock(ptep);
403
-
entry = *ptep;
404
-
/* Check pte entry after all locks have been acquired */
405
-
pte_i = pte_val(entry) & _PAGE_INVALID;
406
-
pte_p = pte_val(entry) & _PAGE_PROTECT;
407
-
if ((pte_i && (prot != PROT_NONE)) ||
408
-
(pte_p && (prot & PROT_WRITE))) {
409
-
pgste_set_unlock(ptep, pgste);
410
-
return -EAGAIN;
411
-
}
412
-
/* Change access rights and set pgste bit */
413
-
nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
414
-
if (prot == PROT_NONE && !pte_i) {
415
-
ptep_flush_direct(mm, addr, ptep, nodat);
416
-
pgste = pgste_update_all(entry, pgste, mm);
417
-
entry = set_pte_bit(entry, __pgprot(_PAGE_INVALID));
418
-
}
419
-
if (prot == PROT_READ && !pte_p) {
420
-
ptep_flush_direct(mm, addr, ptep, nodat);
421
-
entry = clear_pte_bit(entry, __pgprot(_PAGE_INVALID));
422
-
entry = set_pte_bit(entry, __pgprot(_PAGE_PROTECT));
423
-
}
424
-
pgste = set_pgste_bit(pgste, bit);
425
-
pgste = pgste_set_pte(ptep, pgste, entry);
426
-
pgste_set_unlock(ptep, pgste);
427
-
return 0;
428
-
}
429
-
430
-
int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr,
431
-
pte_t *sptep, pte_t *tptep, pte_t pte)
432
-
{
433
-
pgste_t spgste, tpgste;
434
-
pte_t spte, tpte;
435
-
int rc = -EAGAIN;
436
-
437
-
if (!(pte_val(*tptep) & _PAGE_INVALID))
438
-
return 0; /* already shadowed */
439
-
spgste = pgste_get_lock(sptep);
440
-
spte = *sptep;
441
-
if (!(pte_val(spte) & _PAGE_INVALID) &&
442
-
!((pte_val(spte) & _PAGE_PROTECT) &&
443
-
!(pte_val(pte) & _PAGE_PROTECT))) {
444
-
spgste = set_pgste_bit(spgste, PGSTE_VSIE_BIT);
445
-
tpgste = pgste_get_lock(tptep);
446
-
tpte = __pte((pte_val(spte) & PAGE_MASK) |
447
-
(pte_val(pte) & _PAGE_PROTECT));
448
-
/* don't touch the storage key - it belongs to parent pgste */
449
-
tpgste = pgste_set_pte(tptep, tpgste, tpte);
450
-
pgste_set_unlock(tptep, tpgste);
451
-
rc = 1;
452
-
}
453
-
pgste_set_unlock(sptep, spgste);
454
-
return rc;
455
-
}
456
-
457
-
void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep)
458
-
{
459
-
pgste_t pgste;
460
-
int nodat;
461
-
462
-
pgste = pgste_get_lock(ptep);
463
-
/* notifier is called by the caller */
464
-
nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
465
-
ptep_flush_direct(mm, saddr, ptep, nodat);
466
-
/* don't touch the storage key - it belongs to parent pgste */
467
-
pgste = pgste_set_pte(ptep, pgste, __pte(_PAGE_INVALID));
468
-
pgste_set_unlock(ptep, pgste);
469
-
}
470
-
471
-
static void ptep_zap_softleaf_entry(struct mm_struct *mm, softleaf_t entry)
472
-
{
473
-
if (softleaf_is_swap(entry))
474
-
dec_mm_counter(mm, MM_SWAPENTS);
475
-
else if (softleaf_is_migration(entry)) {
476
-
struct folio *folio = softleaf_to_folio(entry);
477
-
478
-
dec_mm_counter(mm, mm_counter(folio));
479
-
}
480
-
free_swap_and_cache(entry);
481
-
}
482
-
483
-
void ptep_zap_unused(struct mm_struct *mm, unsigned long addr,
484
-
pte_t *ptep, int reset)
485
-
{
486
-
unsigned long pgstev;
487
-
pgste_t pgste;
488
-
pte_t pte;
489
-
490
-
/* Zap unused and logically-zero pages */
491
-
preempt_disable();
492
-
pgste = pgste_get_lock(ptep);
493
-
pgstev = pgste_val(pgste);
494
-
pte = *ptep;
495
-
if (!reset && pte_swap(pte) &&
496
-
((pgstev & _PGSTE_GPS_USAGE_MASK) == _PGSTE_GPS_USAGE_UNUSED ||
497
-
(pgstev & _PGSTE_GPS_ZERO))) {
498
-
ptep_zap_softleaf_entry(mm, softleaf_from_pte(pte));
499
-
pte_clear(mm, addr, ptep);
500
-
}
501
-
if (reset)
502
-
pgste = clear_pgste_bit(pgste, _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT);
503
-
pgste_set_unlock(ptep, pgste);
504
-
preempt_enable();
505
-
}
506
-
507
-
void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
508
-
{
509
-
unsigned long ptev;
510
-
pgste_t pgste;
511
-
512
-
/* Clear storage key ACC and F, but set R/C */
513
-
preempt_disable();
514
-
pgste = pgste_get_lock(ptep);
515
-
pgste = clear_pgste_bit(pgste, PGSTE_ACC_BITS | PGSTE_FP_BIT);
516
-
pgste = set_pgste_bit(pgste, PGSTE_GR_BIT | PGSTE_GC_BIT);
517
-
ptev = pte_val(*ptep);
518
-
if (!(ptev & _PAGE_INVALID) && (ptev & _PAGE_WRITE))
519
-
page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 0);
520
-
pgste_set_unlock(ptep, pgste);
521
-
preempt_enable();
522
-
}
523
-
524
-
/*
525
-
* Test and reset if a guest page is dirty
526
-
*/
527
-
bool ptep_test_and_clear_uc(struct mm_struct *mm, unsigned long addr,
528
-
pte_t *ptep)
529
-
{
530
-
pgste_t pgste;
531
-
pte_t pte;
532
-
bool dirty;
533
-
int nodat;
534
-
535
-
pgste = pgste_get_lock(ptep);
536
-
dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT);
537
-
pgste = clear_pgste_bit(pgste, PGSTE_UC_BIT);
538
-
pte = *ptep;
539
-
if (dirty && (pte_val(pte) & _PAGE_PRESENT)) {
540
-
pgste = pgste_pte_notify(mm, addr, ptep, pgste);
541
-
nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
542
-
ptep_ipte_global(mm, addr, ptep, nodat);
543
-
if (machine_has_esop() || !(pte_val(pte) & _PAGE_WRITE))
544
-
pte = set_pte_bit(pte, __pgprot(_PAGE_PROTECT));
545
-
else
546
-
pte = set_pte_bit(pte, __pgprot(_PAGE_INVALID));
547
-
set_pte(ptep, pte);
548
-
}
549
-
pgste_set_unlock(ptep, pgste);
550
-
return dirty;
551
-
}
552
-
EXPORT_SYMBOL_GPL(ptep_test_and_clear_uc);
553
-
554
-
int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
555
-
unsigned char key, bool nq)
556
-
{
557
-
unsigned long keyul, paddr;
558
-
spinlock_t *ptl;
559
-
pgste_t old, new;
560
-
pmd_t *pmdp;
561
-
pte_t *ptep;
562
-
563
-
/*
564
-
* If we don't have a PTE table and if there is no huge page mapped,
565
-
* we can ignore attempts to set the key to 0, because it already is 0.
566
-
*/
567
-
switch (pmd_lookup(mm, addr, &pmdp)) {
568
-
case -ENOENT:
569
-
return key ? -EFAULT : 0;
570
-
case 0:
571
-
break;
572
-
default:
573
-
return -EFAULT;
574
-
}
575
-
again:
576
-
ptl = pmd_lock(mm, pmdp);
577
-
if (!pmd_present(*pmdp)) {
578
-
spin_unlock(ptl);
579
-
return key ? -EFAULT : 0;
580
-
}
581
-
582
-
if (pmd_leaf(*pmdp)) {
583
-
paddr = pmd_val(*pmdp) & HPAGE_MASK;
584
-
paddr |= addr & ~HPAGE_MASK;
585
-
/*
586
-
* Huge pmds need quiescing operations, they are
587
-
* always mapped.
588
-
*/
589
-
page_set_storage_key(paddr, key, 1);
590
-
spin_unlock(ptl);
591
-
return 0;
592
-
}
593
-
spin_unlock(ptl);
594
-
595
-
ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
596
-
if (!ptep)
597
-
goto again;
598
-
new = old = pgste_get_lock(ptep);
599
-
new = clear_pgste_bit(new, PGSTE_GR_BIT | PGSTE_GC_BIT |
600
-
PGSTE_ACC_BITS | PGSTE_FP_BIT);
601
-
keyul = (unsigned long) key;
602
-
new = set_pgste_bit(new, (keyul & (_PAGE_CHANGED | _PAGE_REFERENCED)) << 48);
603
-
new = set_pgste_bit(new, (keyul & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56);
604
-
if (!(pte_val(*ptep) & _PAGE_INVALID)) {
605
-
unsigned long bits, skey;
606
-
607
-
paddr = pte_val(*ptep) & PAGE_MASK;
608
-
skey = (unsigned long) page_get_storage_key(paddr);
609
-
bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
610
-
skey = key & (_PAGE_ACC_BITS | _PAGE_FP_BIT);
611
-
/* Set storage key ACC and FP */
612
-
page_set_storage_key(paddr, skey, !nq);
613
-
/* Merge host changed & referenced into pgste */
614
-
new = set_pgste_bit(new, bits << 52);
615
-
}
616
-
/* changing the guest storage key is considered a change of the page */
617
-
if ((pgste_val(new) ^ pgste_val(old)) &
618
-
(PGSTE_ACC_BITS | PGSTE_FP_BIT | PGSTE_GR_BIT | PGSTE_GC_BIT))
619
-
new = set_pgste_bit(new, PGSTE_UC_BIT);
620
-
621
-
pgste_set_unlock(ptep, new);
622
-
pte_unmap_unlock(ptep, ptl);
623
-
return 0;
624
-
}
625
-
EXPORT_SYMBOL(set_guest_storage_key);
626
-
627
-
/*
628
-
* Conditionally set a guest storage key (handling csske).
629
-
* oldkey will be updated when either mr or mc is set and a pointer is given.
630
-
*
631
-
* Returns 0 if a guests storage key update wasn't necessary, 1 if the guest
632
-
* storage key was updated and -EFAULT on access errors.
633
-
*/
634
-
int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
635
-
unsigned char key, unsigned char *oldkey,
636
-
bool nq, bool mr, bool mc)
637
-
{
638
-
unsigned char tmp, mask = _PAGE_ACC_BITS | _PAGE_FP_BIT;
639
-
int rc;
640
-
641
-
/* we can drop the pgste lock between getting and setting the key */
642
-
if (mr | mc) {
643
-
rc = get_guest_storage_key(current->mm, addr, &tmp);
644
-
if (rc)
645
-
return rc;
646
-
if (oldkey)
647
-
*oldkey = tmp;
648
-
if (!mr)
649
-
mask |= _PAGE_REFERENCED;
650
-
if (!mc)
651
-
mask |= _PAGE_CHANGED;
652
-
if (!((tmp ^ key) & mask))
653
-
return 0;
654
-
}
655
-
rc = set_guest_storage_key(current->mm, addr, key, nq);
656
-
return rc < 0 ? rc : 1;
657
-
}
658
-
EXPORT_SYMBOL(cond_set_guest_storage_key);
659
-
660
-
/*
661
-
* Reset a guest reference bit (rrbe), returning the reference and changed bit.
662
-
*
663
-
* Returns < 0 in case of error, otherwise the cc to be reported to the guest.
664
-
*/
665
-
int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr)
666
-
{
667
-
spinlock_t *ptl;
668
-
unsigned long paddr;
669
-
pgste_t old, new;
670
-
pmd_t *pmdp;
671
-
pte_t *ptep;
672
-
int cc = 0;
673
-
674
-
/*
675
-
* If we don't have a PTE table and if there is no huge page mapped,
676
-
* the storage key is 0 and there is nothing for us to do.
677
-
*/
678
-
switch (pmd_lookup(mm, addr, &pmdp)) {
679
-
case -ENOENT:
680
-
return 0;
681
-
case 0:
682
-
break;
683
-
default:
684
-
return -EFAULT;
685
-
}
686
-
again:
687
-
ptl = pmd_lock(mm, pmdp);
688
-
if (!pmd_present(*pmdp)) {
689
-
spin_unlock(ptl);
690
-
return 0;
691
-
}
692
-
693
-
if (pmd_leaf(*pmdp)) {
694
-
paddr = pmd_val(*pmdp) & HPAGE_MASK;
695
-
paddr |= addr & ~HPAGE_MASK;
696
-
cc = page_reset_referenced(paddr);
697
-
spin_unlock(ptl);
698
-
return cc;
699
-
}
700
-
spin_unlock(ptl);
701
-
702
-
ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
703
-
if (!ptep)
704
-
goto again;
705
-
new = old = pgste_get_lock(ptep);
706
-
/* Reset guest reference bit only */
707
-
new = clear_pgste_bit(new, PGSTE_GR_BIT);
708
-
709
-
if (!(pte_val(*ptep) & _PAGE_INVALID)) {
710
-
paddr = pte_val(*ptep) & PAGE_MASK;
711
-
cc = page_reset_referenced(paddr);
712
-
/* Merge real referenced bit into host-set */
713
-
new = set_pgste_bit(new, ((unsigned long)cc << 53) & PGSTE_HR_BIT);
714
-
}
715
-
/* Reflect guest's logical view, not physical */
716
-
cc |= (pgste_val(old) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 49;
717
-
/* Changing the guest storage key is considered a change of the page */
718
-
if ((pgste_val(new) ^ pgste_val(old)) & PGSTE_GR_BIT)
719
-
new = set_pgste_bit(new, PGSTE_UC_BIT);
720
-
721
-
pgste_set_unlock(ptep, new);
722
-
pte_unmap_unlock(ptep, ptl);
723
-
return cc;
724
-
}
725
-
EXPORT_SYMBOL(reset_guest_reference_bit);
726
-
727
-
int get_guest_storage_key(struct mm_struct *mm, unsigned long addr,
728
-
unsigned char *key)
729
-
{
730
-
unsigned long paddr;
731
-
spinlock_t *ptl;
732
-
pgste_t pgste;
733
-
pmd_t *pmdp;
734
-
pte_t *ptep;
735
-
736
-
/*
737
-
* If we don't have a PTE table and if there is no huge page mapped,
738
-
* the storage key is 0.
739
-
*/
740
-
*key = 0;
741
-
742
-
switch (pmd_lookup(mm, addr, &pmdp)) {
743
-
case -ENOENT:
744
-
return 0;
745
-
case 0:
746
-
break;
747
-
default:
748
-
return -EFAULT;
749
-
}
750
-
again:
751
-
ptl = pmd_lock(mm, pmdp);
752
-
if (!pmd_present(*pmdp)) {
753
-
spin_unlock(ptl);
754
-
return 0;
755
-
}
756
-
757
-
if (pmd_leaf(*pmdp)) {
758
-
paddr = pmd_val(*pmdp) & HPAGE_MASK;
759
-
paddr |= addr & ~HPAGE_MASK;
760
-
*key = page_get_storage_key(paddr);
761
-
spin_unlock(ptl);
762
-
return 0;
763
-
}
764
-
spin_unlock(ptl);
765
-
766
-
ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
767
-
if (!ptep)
768
-
goto again;
769
-
pgste = pgste_get_lock(ptep);
770
-
*key = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56;
771
-
paddr = pte_val(*ptep) & PAGE_MASK;
772
-
if (!(pte_val(*ptep) & _PAGE_INVALID))
773
-
*key = page_get_storage_key(paddr);
774
-
/* Reflect guest's logical view, not physical */
775
-
*key |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48;
776
-
pgste_set_unlock(ptep, pgste);
777
-
pte_unmap_unlock(ptep, ptl);
778
-
return 0;
779
-
}
780
-
EXPORT_SYMBOL(get_guest_storage_key);
781
-
782
-
/**
783
-
* pgste_perform_essa - perform ESSA actions on the PGSTE.
784
-
* @mm: the memory context. It must have PGSTEs, no check is performed here!
785
-
* @hva: the host virtual address of the page whose PGSTE is to be processed
786
-
* @orc: the specific action to perform, see the ESSA_SET_* macros.
787
-
* @oldpte: the PTE will be saved there if the pointer is not NULL.
788
-
* @oldpgste: the old PGSTE will be saved there if the pointer is not NULL.
789
-
*
790
-
* Return: 1 if the page is to be added to the CBRL, otherwise 0,
791
-
* or < 0 in case of error. -EINVAL is returned for invalid values
792
-
* of orc, -EFAULT for invalid addresses.
793
-
*/
794
-
int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc,
795
-
unsigned long *oldpte, unsigned long *oldpgste)
796
-
{
797
-
struct vm_area_struct *vma;
798
-
unsigned long pgstev;
799
-
spinlock_t *ptl;
800
-
pgste_t pgste;
801
-
pte_t *ptep;
802
-
int res = 0;
803
-
804
-
WARN_ON_ONCE(orc > ESSA_MAX);
805
-
if (unlikely(orc > ESSA_MAX))
806
-
return -EINVAL;
807
-
808
-
vma = vma_lookup(mm, hva);
809
-
if (!vma || is_vm_hugetlb_page(vma))
810
-
return -EFAULT;
811
-
ptep = get_locked_pte(mm, hva, &ptl);
812
-
if (unlikely(!ptep))
813
-
return -EFAULT;
814
-
pgste = pgste_get_lock(ptep);
815
-
pgstev = pgste_val(pgste);
816
-
if (oldpte)
817
-
*oldpte = pte_val(*ptep);
818
-
if (oldpgste)
819
-
*oldpgste = pgstev;
820
-
821
-
switch (orc) {
822
-
case ESSA_GET_STATE:
823
-
break;
824
-
case ESSA_SET_STABLE:
825
-
pgstev &= ~(_PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT);
826
-
pgstev |= _PGSTE_GPS_USAGE_STABLE;
827
-
break;
828
-
case ESSA_SET_UNUSED:
829
-
pgstev &= ~_PGSTE_GPS_USAGE_MASK;
830
-
pgstev |= _PGSTE_GPS_USAGE_UNUSED;
831
-
if (pte_val(*ptep) & _PAGE_INVALID)
832
-
res = 1;
833
-
break;
834
-
case ESSA_SET_VOLATILE:
835
-
pgstev &= ~_PGSTE_GPS_USAGE_MASK;
836
-
pgstev |= _PGSTE_GPS_USAGE_VOLATILE;
837
-
if (pte_val(*ptep) & _PAGE_INVALID)
838
-
res = 1;
839
-
break;
840
-
case ESSA_SET_POT_VOLATILE:
841
-
pgstev &= ~_PGSTE_GPS_USAGE_MASK;
842
-
if (!(pte_val(*ptep) & _PAGE_INVALID)) {
843
-
pgstev |= _PGSTE_GPS_USAGE_POT_VOLATILE;
844
-
break;
845
-
}
846
-
if (pgstev & _PGSTE_GPS_ZERO) {
847
-
pgstev |= _PGSTE_GPS_USAGE_VOLATILE;
848
-
break;
849
-
}
850
-
if (!(pgstev & PGSTE_GC_BIT)) {
851
-
pgstev |= _PGSTE_GPS_USAGE_VOLATILE;
852
-
res = 1;
853
-
break;
854
-
}
855
-
break;
856
-
case ESSA_SET_STABLE_RESIDENT:
857
-
pgstev &= ~_PGSTE_GPS_USAGE_MASK;
858
-
pgstev |= _PGSTE_GPS_USAGE_STABLE;
859
-
/*
860
-
* Since the resident state can go away any time after this
861
-
* call, we will not make this page resident. We can revisit
862
-
* this decision if a guest will ever start using this.
863
-
*/
864
-
break;
865
-
case ESSA_SET_STABLE_IF_RESIDENT:
866
-
if (!(pte_val(*ptep) & _PAGE_INVALID)) {
867
-
pgstev &= ~_PGSTE_GPS_USAGE_MASK;
868
-
pgstev |= _PGSTE_GPS_USAGE_STABLE;
869
-
}
870
-
break;
871
-
case ESSA_SET_STABLE_NODAT:
872
-
pgstev &= ~_PGSTE_GPS_USAGE_MASK;
873
-
pgstev |= _PGSTE_GPS_USAGE_STABLE | _PGSTE_GPS_NODAT;
874
-
break;
875
-
default:
876
-
/* we should never get here! */
877
-
break;
878
-
}
879
-
/* If we are discarding a page, set it to logical zero */
880
-
if (res)
881
-
pgstev |= _PGSTE_GPS_ZERO;
882
-
883
-
pgste = __pgste(pgstev);
884
-
pgste_set_unlock(ptep, pgste);
885
-
pte_unmap_unlock(ptep, ptl);
886
-
return res;
887
-
}
888
-
EXPORT_SYMBOL(pgste_perform_essa);
889
-
890
-
/**
891
-
* set_pgste_bits - set specific PGSTE bits.
892
-
* @mm: the memory context. It must have PGSTEs, no check is performed here!
893
-
* @hva: the host virtual address of the page whose PGSTE is to be processed
894
-
* @bits: a bitmask representing the bits that will be touched
895
-
* @value: the values of the bits to be written. Only the bits in the mask
896
-
* will be written.
897
-
*
898
-
* Return: 0 on success, < 0 in case of error.
899
-
*/
900
-
int set_pgste_bits(struct mm_struct *mm, unsigned long hva,
901
-
unsigned long bits, unsigned long value)
902
-
{
903
-
struct vm_area_struct *vma;
904
-
spinlock_t *ptl;
905
-
pgste_t new;
906
-
pte_t *ptep;
907
-
908
-
vma = vma_lookup(mm, hva);
909
-
if (!vma || is_vm_hugetlb_page(vma))
910
-
return -EFAULT;
911
-
ptep = get_locked_pte(mm, hva, &ptl);
912
-
if (unlikely(!ptep))
913
-
return -EFAULT;
914
-
new = pgste_get_lock(ptep);
915
-
916
-
new = clear_pgste_bit(new, bits);
917
-
new = set_pgste_bit(new, value & bits);
918
-
919
-
pgste_set_unlock(ptep, new);
920
-
pte_unmap_unlock(ptep, ptl);
921
-
return 0;
922
-
}
923
-
EXPORT_SYMBOL(set_pgste_bits);
924
-
925
-
/**
926
-
* get_pgste - get the current PGSTE for the given address.
927
-
* @mm: the memory context. It must have PGSTEs, no check is performed here!
928
-
* @hva: the host virtual address of the page whose PGSTE is to be processed
929
-
* @pgstep: will be written with the current PGSTE for the given address.
930
-
*
931
-
* Return: 0 on success, < 0 in case of error.
932
-
*/
933
-
int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep)
934
-
{
935
-
struct vm_area_struct *vma;
936
-
spinlock_t *ptl;
937
-
pte_t *ptep;
938
-
939
-
vma = vma_lookup(mm, hva);
940
-
if (!vma || is_vm_hugetlb_page(vma))
941
-
return -EFAULT;
942
-
ptep = get_locked_pte(mm, hva, &ptl);
943
-
if (unlikely(!ptep))
944
-
return -EFAULT;
945
-
*pgstep = pgste_val(pgste_get(ptep));
946
-
pte_unmap_unlock(ptep, ptl);
947
-
return 0;
948
-
}
949
-
EXPORT_SYMBOL(get_pgste);
950
-
#endif
-9
mm/khugepaged.c
-9
mm/khugepaged.c
···
343
343
{
344
344
switch (advice) {
345
345
case MADV_HUGEPAGE:
346
-
#ifdef CONFIG_S390
347
-
/*
348
-
* qemu blindly sets MADV_HUGEPAGE on all allocations, but s390
349
-
* can't handle this properly after s390_enable_sie, so we simply
350
-
* ignore the madvise to prevent qemu from causing a SIGSEGV.
351
-
*/
352
-
if (mm_has_pgste(vma->vm_mm))
353
-
return 0;
354
-
#endif
355
346
*vm_flags &= ~VM_NOHUGEPAGE;
356
347
*vm_flags |= VM_HUGEPAGE;
357
348
/*