diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h index 972d9b8d269f6e863f2922374c0bf8326ae96ede..ff5892172dc2a2eb320ea45814f2fbd07c0ca5df 100644 --- a/arch/arm64/include/asm/tlb.h +++ b/arch/arm64/include/asm/tlb.h @@ -89,7 +89,9 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp, unsigned long addr) { - tlb_remove_ptdesc(tlb, virt_to_ptdesc(pudp)); + struct ptdesc *ptdesc = virt_to_ptdesc(pudp); + + tlb_remove_ptdesc(tlb, ptdesc); } #endif diff --git a/arch/loongarch/include/asm/pgalloc.h b/arch/loongarch/include/asm/pgalloc.h index c9f9895f237d974029fd9dc4242f1bff4610aa15..a7b9c9e73593d2e4a7d79134ad3b70f3415787c4 100644 --- a/arch/loongarch/include/asm/pgalloc.h +++ b/arch/loongarch/include/asm/pgalloc.h @@ -95,6 +95,7 @@ static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address) if (!ptdesc) return NULL; + pagetable_pud_ctor(ptdesc); pud = ptdesc_address(ptdesc); pud_init(pud); diff --git a/arch/mips/include/asm/pgalloc.h b/arch/mips/include/asm/pgalloc.h index 40e40a7eb94af582c690f72ec52f99efe2704df3..f4440edcd8fe2eec7529362c004143ad6fe05c91 100644 --- a/arch/mips/include/asm/pgalloc.h +++ b/arch/mips/include/asm/pgalloc.h @@ -95,6 +95,7 @@ static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address) if (!ptdesc) return NULL; + pagetable_pud_ctor(ptdesc); pud = ptdesc_address(ptdesc); pud_init(pud); diff --git a/arch/riscv/include/asm/pgalloc.h b/arch/riscv/include/asm/pgalloc.h index 9757a1813c7d02179b3c5afd6a2f5417f1ee3891..7e672f4e2e16cb9b2364aa1b42ed6e65efe97a3b 100644 --- a/arch/riscv/include/asm/pgalloc.h +++ b/arch/riscv/include/asm/pgalloc.h @@ -90,16 +90,23 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud) #define p4d_alloc_one p4d_alloc_one static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr) { + p4d_t *p4d; gfp_t gfp = GFP_PGTABLE_USER; if (mm == &init_mm) gfp = GFP_PGTABLE_KERNEL; - return (p4d_t *)get_zeroed_page(gfp); + p4d = (p4d_t *)get_zeroed_page(gfp); + if (!p4d) + return NULL; + + pagetable_p4d_ctor(virt_to_ptdesc(p4d)); + return p4d; } static inline void __p4d_free(struct mm_struct *mm, p4d_t *p4d) { BUG_ON((unsigned long)p4d & (PAGE_SIZE-1)); + pagetable_dtor(virt_to_ptdesc(p4d)); free_page((unsigned long)p4d); } @@ -141,7 +148,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm) #define __pte_free_tlb(tlb, pte, buf) \ do { \ - pagetable_pte_dtor(page_ptdesc(pte)); \ + pagetable_dtor(page_ptdesc(pte)); \ tlb_remove_page_ptdesc((tlb), page_ptdesc(pte));\ } while (0) #endif /* CONFIG_MMU */ diff --git a/arch/x86/include/asm/pgalloc.h b/arch/x86/include/asm/pgalloc.h index c7ec5bb88334eab119ccf78002be2e7679291113..a3ed92a87fad0c24e197e6ad61bafe7df3df9c12 100644 --- a/arch/x86/include/asm/pgalloc.h +++ b/arch/x86/include/asm/pgalloc.h @@ -149,11 +149,17 @@ static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4 static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr) { + p4d_t *p4d; gfp_t gfp = GFP_KERNEL_ACCOUNT; if (mm == &init_mm) gfp &= ~__GFP_ACCOUNT; - return (p4d_t *)get_zeroed_page(gfp); + p4d = (p4d_t *)get_zeroed_page(gfp); + if (!p4d) + return NULL; + + pagetable_p4d_ctor(virt_to_ptdesc(p4d)); + return p4d; } static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d) @@ -162,6 +168,7 @@ static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d) return; BUG_ON((unsigned long)p4d & (PAGE_SIZE-1)); + pagetable_dtor(virt_to_ptdesc(p4d)); free_page((unsigned long)p4d); } diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index 17071b7114acf5ff20d4d1e4f2c6b1cb12993187..1362be73a6d0ffb7a9199893523a9c4fbf140e15 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -84,15 +84,19 @@ void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd) #if CONFIG_PGTABLE_LEVELS > 3 void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud) { + struct ptdesc *ptdesc = virt_to_ptdesc(pud); + paravirt_release_pud(__pa(pud) >> PAGE_SHIFT); - paravirt_tlb_remove_table(tlb, virt_to_ptdesc(pud)); + paravirt_tlb_remove_table(tlb, ptdesc); } #if CONFIG_PGTABLE_LEVELS > 4 void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d) { + struct ptdesc *ptdesc = virt_to_ptdesc(p4d); + paravirt_release_p4d(__pa(p4d) >> PAGE_SHIFT); - paravirt_tlb_remove_table(tlb, virt_to_ptdesc(p4d)); + paravirt_tlb_remove_table(tlb, ptdesc); } #endif /* CONFIG_PGTABLE_LEVELS > 4 */ #endif /* CONFIG_PGTABLE_LEVELS > 3 */ diff --git a/include/asm-generic/pgalloc.h b/include/asm-generic/pgalloc.h index 07698be4c7da7f6d2e8af07d3612bf7b9900ba0f..35d0a541ca2d538acc1c49d1bc7feb5f941db36a 100644 --- a/include/asm-generic/pgalloc.h +++ b/include/asm-generic/pgalloc.h @@ -207,6 +207,8 @@ static inline pud_t *__pud_alloc_one(struct mm_struct *mm, unsigned long addr) ptdesc = pagetable_alloc(gfp, 0); if (!ptdesc) return NULL; + + pagetable_pud_ctor(ptdesc); return ptdesc_address(ptdesc); } @@ -228,8 +230,11 @@ static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) static inline void __pud_free(struct mm_struct *mm, pud_t *pud) { + struct ptdesc *ptdesc = virt_to_ptdesc(pud); + BUG_ON((unsigned long)pud & (PAGE_SIZE-1)); - pagetable_free(virt_to_ptdesc(pud)); + pagetable_dtor(ptdesc); + pagetable_free(ptdesc); } #ifndef __HAVE_ARCH_PUD_FREE diff --git a/include/linux/mm.h b/include/linux/mm.h index c6a5a7af7ab1dd37cf9c992198d06e5928af5334..ea1333659fde8e213ef8e736bcdc3c4db87837b0 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3223,6 +3223,22 @@ static inline spinlock_t *pud_lock(struct mm_struct *mm, pud_t *pud) return ptl; } +static inline void pagetable_pud_ctor(struct ptdesc *ptdesc) +{ + struct folio *folio = ptdesc_folio(ptdesc); + + __folio_set_pgtable(folio); + lruvec_stat_add_folio(folio, NR_PAGETABLE); +} + +static inline void pagetable_p4d_ctor(struct ptdesc *ptdesc) +{ + struct folio *folio = ptdesc_folio(ptdesc); + + __folio_set_pgtable(folio); + lruvec_stat_add_folio(folio, NR_PAGETABLE); +} + extern void __init pagecache_init(void); extern void free_initmem(void);