Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'page-table-check-support' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux into features
Add s390 support for CONFIG_PAGE_TABLE_CHECK.
* 'page-table-check-support' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux:
s390: Enable page table check for debug_defconfig
s390/pgtable: Add s390 support for page table check
s390/pgtable: Use set_pmd_bit() to invalidate PMD entry
mm/page_table_check: Pass mm_struct to pxx_user_accessible_page()
Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>
+80
-34
+3
-3
arch/arm64/include/asm/pgtable.h
+3
-3
arch/arm64/include/asm/pgtable.h
···
1263
1263
#endif
1264
1264
1265
1265
#ifdef CONFIG_PAGE_TABLE_CHECK
1266
-
static inline bool pte_user_accessible_page(pte_t pte, unsigned long addr)
1266
+
static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
1267
1267
{
1268
1268
return pte_valid(pte) && (pte_user(pte) || pte_user_exec(pte));
1269
1269
}
1270
1270
1271
-
static inline bool pmd_user_accessible_page(pmd_t pmd, unsigned long addr)
1271
+
static inline bool pmd_user_accessible_page(struct mm_struct *mm, unsigned long addr, pmd_t pmd)
1272
1272
{
1273
1273
return pmd_valid(pmd) && !pmd_table(pmd) && (pmd_user(pmd) || pmd_user_exec(pmd));
1274
1274
}
1275
1275
1276
-
static inline bool pud_user_accessible_page(pud_t pud, unsigned long addr)
1276
+
static inline bool pud_user_accessible_page(struct mm_struct *mm, unsigned long addr, pud_t pud)
1277
1277
{
1278
1278
return pud_valid(pud) && !pud_table(pud) && (pud_user(pud) || pud_user_exec(pud));
1279
1279
}
+1
-1
arch/powerpc/include/asm/book3s/32/pgtable.h
+1
-1
arch/powerpc/include/asm/book3s/32/pgtable.h
···
438
438
return true;
439
439
}
440
440
441
-
static inline bool pte_user_accessible_page(pte_t pte, unsigned long addr)
441
+
static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
442
442
{
443
443
return pte_present(pte) && !is_kernel_addr(addr);
444
444
}
+5
-5
arch/powerpc/include/asm/book3s/64/pgtable.h
+5
-5
arch/powerpc/include/asm/book3s/64/pgtable.h
···
549
549
return arch_pte_access_permitted(pte_val(pte), write, 0);
550
550
}
551
551
552
-
static inline bool pte_user_accessible_page(pte_t pte, unsigned long addr)
552
+
static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
553
553
{
554
554
return pte_present(pte) && pte_user(pte);
555
555
}
···
925
925
}
926
926
927
927
#define pud_user_accessible_page pud_user_accessible_page
928
-
static inline bool pud_user_accessible_page(pud_t pud, unsigned long addr)
928
+
static inline bool pud_user_accessible_page(struct mm_struct *mm, unsigned long addr, pud_t pud)
929
929
{
930
-
return pud_leaf(pud) && pte_user_accessible_page(pud_pte(pud), addr);
930
+
return pud_leaf(pud) && pte_user_accessible_page(mm, addr, pud_pte(pud));
931
931
}
932
932
933
933
#define __p4d_raw(x) ((p4d_t) { __pgd_raw(x) })
···
1096
1096
}
1097
1097
1098
1098
#define pmd_user_accessible_page pmd_user_accessible_page
1099
-
static inline bool pmd_user_accessible_page(pmd_t pmd, unsigned long addr)
1099
+
static inline bool pmd_user_accessible_page(struct mm_struct *mm, unsigned long addr, pmd_t pmd)
1100
1100
{
1101
-
return pmd_leaf(pmd) && pte_user_accessible_page(pmd_pte(pmd), addr);
1101
+
return pmd_leaf(pmd) && pte_user_accessible_page(mm, addr, pmd_pte(pmd));
1102
1102
}
1103
1103
1104
1104
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+1
-1
arch/powerpc/include/asm/nohash/pgtable.h
+1
-1
arch/powerpc/include/asm/nohash/pgtable.h
···
249
249
return true;
250
250
}
251
251
252
-
static inline bool pte_user_accessible_page(pte_t pte, unsigned long addr)
252
+
static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
253
253
{
254
254
return pte_present(pte) && !is_kernel_addr(addr);
255
255
}
+2
-2
arch/powerpc/include/asm/pgtable.h
+2
-2
arch/powerpc/include/asm/pgtable.h
···
205
205
#endif /* CONFIG_PPC64 */
206
206
207
207
#ifndef pmd_user_accessible_page
208
-
#define pmd_user_accessible_page(pmd, addr) false
208
+
#define pmd_user_accessible_page(mm, addr, pmd) false
209
209
#endif
210
210
211
211
#ifndef pud_user_accessible_page
212
-
#define pud_user_accessible_page(pud, addr) false
212
+
#define pud_user_accessible_page(mm, addr, pud) false
213
213
#endif
214
214
215
215
#endif /* __ASSEMBLER__ */
+3
-3
arch/riscv/include/asm/pgtable.h
+3
-3
arch/riscv/include/asm/pgtable.h
···
984
984
}
985
985
986
986
#ifdef CONFIG_PAGE_TABLE_CHECK
987
-
static inline bool pte_user_accessible_page(pte_t pte, unsigned long addr)
987
+
static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
988
988
{
989
989
return pte_present(pte) && pte_user(pte);
990
990
}
991
991
992
-
static inline bool pmd_user_accessible_page(pmd_t pmd, unsigned long addr)
992
+
static inline bool pmd_user_accessible_page(struct mm_struct *mm, unsigned long addr, pmd_t pmd)
993
993
{
994
994
return pmd_leaf(pmd) && pmd_user(pmd);
995
995
}
996
996
997
-
static inline bool pud_user_accessible_page(pud_t pud, unsigned long addr)
997
+
static inline bool pud_user_accessible_page(struct mm_struct *mm, unsigned long addr, pud_t pud)
998
998
{
999
999
return pud_leaf(pud) && pud_user(pud);
1000
1000
}
+1
arch/s390/Kconfig
+1
arch/s390/Kconfig
···
154
154
select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && CC_IS_CLANG
155
155
select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS
156
156
select ARCH_SUPPORTS_NUMA_BALANCING
157
+
select ARCH_SUPPORTS_PAGE_TABLE_CHECK
157
158
select ARCH_SUPPORTS_PER_VMA_LOCK
158
159
select ARCH_USE_BUILTIN_BSWAP
159
160
select ARCH_USE_CMPXCHG_LOCKREF
+2
arch/s390/configs/debug_defconfig
+2
arch/s390/configs/debug_defconfig
+53
-7
arch/s390/include/asm/pgtable.h
+53
-7
arch/s390/include/asm/pgtable.h
···
16
16
#include <linux/mm_types.h>
17
17
#include <linux/cpufeature.h>
18
18
#include <linux/page-flags.h>
19
+
#include <linux/page_table_check.h>
19
20
#include <linux/radix-tree.h>
20
21
#include <linux/atomic.h>
22
+
#include <linux/mmap_lock.h>
21
23
#include <asm/ctlreg.h>
22
24
#include <asm/bug.h>
23
25
#include <asm/page.h>
···
1192
1190
/* At this point the reference through the mapping is still present */
1193
1191
if (mm_is_protected(mm) && pte_present(res))
1194
1192
WARN_ON_ONCE(uv_convert_from_secure_pte(res));
1193
+
page_table_check_pte_clear(mm, addr, res);
1195
1194
return res;
1196
1195
}
1197
1196
···
1211
1208
/* At this point the reference through the mapping is still present */
1212
1209
if (mm_is_protected(vma->vm_mm) && pte_present(res))
1213
1210
WARN_ON_ONCE(uv_convert_from_secure_pte(res));
1211
+
page_table_check_pte_clear(vma->vm_mm, addr, res);
1214
1212
return res;
1215
1213
}
1216
1214
···
1235
1231
} else {
1236
1232
res = ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID));
1237
1233
}
1234
+
1235
+
page_table_check_pte_clear(mm, addr, res);
1236
+
1238
1237
/* Nothing to do */
1239
1238
if (!mm_is_protected(mm) || !pte_present(res))
1240
1239
return res;
···
1334
1327
{
1335
1328
if (pte_present(entry))
1336
1329
entry = clear_pte_bit(entry, __pgprot(_PAGE_UNUSED));
1330
+
page_table_check_ptes_set(mm, addr, ptep, entry, nr);
1337
1331
for (;;) {
1338
1332
set_pte(ptep, entry);
1339
1333
if (--nr == 0)
···
1711
1703
static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
1712
1704
pmd_t *pmdp, pmd_t entry)
1713
1705
{
1706
+
page_table_check_pmd_set(mm, addr, pmdp, entry);
1714
1707
set_pmd(pmdp, entry);
1715
1708
}
1716
1709
···
1726
1717
static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
1727
1718
unsigned long addr, pmd_t *pmdp)
1728
1719
{
1729
-
return pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
1720
+
pmd_t pmd;
1721
+
1722
+
pmd = pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
1723
+
page_table_check_pmd_clear(mm, addr, pmd);
1724
+
return pmd;
1730
1725
}
1731
1726
1732
1727
#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL
···
1738
1725
unsigned long addr,
1739
1726
pmd_t *pmdp, int full)
1740
1727
{
1728
+
pmd_t pmd;
1729
+
1741
1730
if (full) {
1742
-
pmd_t pmd = *pmdp;
1731
+
pmd = *pmdp;
1743
1732
set_pmd(pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
1733
+
page_table_check_pmd_clear(vma->vm_mm, addr, pmd);
1744
1734
return pmd;
1745
1735
}
1746
-
return pmdp_xchg_lazy(vma->vm_mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
1736
+
pmd = pmdp_xchg_lazy(vma->vm_mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
1737
+
page_table_check_pmd_clear(vma->vm_mm, addr, pmd);
1738
+
return pmd;
1747
1739
}
1748
1740
1749
1741
#define __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
···
1762
1744
static inline pmd_t pmdp_invalidate(struct vm_area_struct *vma,
1763
1745
unsigned long addr, pmd_t *pmdp)
1764
1746
{
1765
-
pmd_t pmd;
1747
+
pmd_t pmd = *pmdp;
1766
1748
1767
-
VM_WARN_ON_ONCE(!pmd_present(*pmdp));
1768
-
pmd = __pmd(pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID);
1769
-
return pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd);
1749
+
VM_WARN_ON_ONCE(!pmd_present(pmd));
1750
+
pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_INVALID));
1751
+
#ifdef CONFIG_PAGE_TABLE_CHECK
1752
+
pmd = clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_READ));
1753
+
#endif
1754
+
page_table_check_pmd_set(vma->vm_mm, addr, pmdp, pmd);
1755
+
pmd = pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd);
1756
+
return pmd;
1770
1757
}
1771
1758
1772
1759
#define __HAVE_ARCH_PMDP_SET_WRPROTECT
···
1805
1782
return cpu_has_edat1() ? 1 : 0;
1806
1783
}
1807
1784
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1785
+
1786
+
#ifdef CONFIG_PAGE_TABLE_CHECK
1787
+
static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
1788
+
{
1789
+
VM_BUG_ON(mm == &init_mm);
1790
+
1791
+
return pte_present(pte);
1792
+
}
1793
+
1794
+
static inline bool pmd_user_accessible_page(struct mm_struct *mm, unsigned long addr, pmd_t pmd)
1795
+
{
1796
+
VM_BUG_ON(mm == &init_mm);
1797
+
1798
+
return pmd_leaf(pmd) && (pmd_val(pmd) & _SEGMENT_ENTRY_READ);
1799
+
}
1800
+
1801
+
static inline bool pud_user_accessible_page(struct mm_struct *mm, unsigned long addr, pud_t pud)
1802
+
{
1803
+
VM_BUG_ON(mm == &init_mm);
1804
+
1805
+
return pud_leaf(pud);
1806
+
}
1807
+
#endif
1808
1808
1809
1809
/*
1810
1810
* 64 bit swap entry format:
+3
-3
arch/x86/include/asm/pgtable.h
+3
-3
arch/x86/include/asm/pgtable.h
···
1680
1680
#endif
1681
1681
1682
1682
#ifdef CONFIG_PAGE_TABLE_CHECK
1683
-
static inline bool pte_user_accessible_page(pte_t pte, unsigned long addr)
1683
+
static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
1684
1684
{
1685
1685
return (pte_val(pte) & _PAGE_PRESENT) && (pte_val(pte) & _PAGE_USER);
1686
1686
}
1687
1687
1688
-
static inline bool pmd_user_accessible_page(pmd_t pmd, unsigned long addr)
1688
+
static inline bool pmd_user_accessible_page(struct mm_struct *mm, unsigned long addr, pmd_t pmd)
1689
1689
{
1690
1690
return pmd_leaf(pmd) && (pmd_val(pmd) & _PAGE_PRESENT) && (pmd_val(pmd) & _PAGE_USER);
1691
1691
}
1692
1692
1693
-
static inline bool pud_user_accessible_page(pud_t pud, unsigned long addr)
1693
+
static inline bool pud_user_accessible_page(struct mm_struct *mm, unsigned long addr, pud_t pud)
1694
1694
{
1695
1695
return pud_leaf(pud) && (pud_val(pud) & _PAGE_PRESENT) && (pud_val(pud) & _PAGE_USER);
1696
1696
}
+6
-9
mm/page_table_check.c
+6
-9
mm/page_table_check.c
···
151
151
if (&init_mm == mm)
152
152
return;
153
153
154
-
if (pte_user_accessible_page(pte, addr)) {
154
+
if (pte_user_accessible_page(mm, addr, pte))
155
155
page_table_check_clear(pte_pfn(pte), PAGE_SIZE >> PAGE_SHIFT);
156
-
}
157
156
}
158
157
EXPORT_SYMBOL(__page_table_check_pte_clear);
159
158
···
162
163
if (&init_mm == mm)
163
164
return;
164
165
165
-
if (pmd_user_accessible_page(pmd, addr)) {
166
+
if (pmd_user_accessible_page(mm, addr, pmd))
166
167
page_table_check_clear(pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT);
167
-
}
168
168
}
169
169
EXPORT_SYMBOL(__page_table_check_pmd_clear);
170
170
···
173
175
if (&init_mm == mm)
174
176
return;
175
177
176
-
if (pud_user_accessible_page(pud, addr)) {
178
+
if (pud_user_accessible_page(mm, addr, pud))
177
179
page_table_check_clear(pud_pfn(pud), PUD_SIZE >> PAGE_SHIFT);
178
-
}
179
180
}
180
181
EXPORT_SYMBOL(__page_table_check_pud_clear);
181
182
···
208
211
209
212
for (i = 0; i < nr; i++)
210
213
__page_table_check_pte_clear(mm, addr + PAGE_SIZE * i, ptep_get(ptep + i));
211
-
if (pte_user_accessible_page(pte, addr))
214
+
if (pte_user_accessible_page(mm, addr, pte))
212
215
page_table_check_set(pte_pfn(pte), nr, pte_write(pte));
213
216
}
214
217
EXPORT_SYMBOL(__page_table_check_ptes_set);
···
238
241
239
242
for (i = 0; i < nr; i++)
240
243
__page_table_check_pmd_clear(mm, addr + PMD_SIZE * i, *(pmdp + i));
241
-
if (pmd_user_accessible_page(pmd, addr))
244
+
if (pmd_user_accessible_page(mm, addr, pmd))
242
245
page_table_check_set(pmd_pfn(pmd), stride * nr, pmd_write(pmd));
243
246
}
244
247
EXPORT_SYMBOL(__page_table_check_pmds_set);
···
254
257
255
258
for (i = 0; i < nr; i++)
256
259
__page_table_check_pud_clear(mm, addr + PUD_SIZE * i, *(pudp + i));
257
-
if (pud_user_accessible_page(pud, addr))
260
+
if (pud_user_accessible_page(mm, addr, pud))
258
261
page_table_check_set(pud_pfn(pud), stride * nr, pud_write(pud));
259
262
}
260
263
EXPORT_SYMBOL(__page_table_check_puds_set);