diff --git a/drivers/char/mem.c b/drivers/char/mem.c index 1052b0f2d4cf86fed49218770a389669e6a3bcf2..ffa78a9f97cc2d8f93ef88c2f1b66395202fb008 100644 --- a/drivers/char/mem.c +++ b/drivers/char/mem.c @@ -23,6 +23,9 @@ #include #include #include +#ifdef CONFIG_MMAP_HUGEPAGE +#include +#endif #include #include #include @@ -395,6 +398,101 @@ static int mmap_mem(struct file *file, struct vm_area_struct *vma) return 0; } +#ifdef CONFIG_MMAP_HUGEPAGE +extern pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page, + int writable); +#define HUGE_PAGE_SZ_2M 0x200000 + +bool is_mmap_hp_range(struct vm_area_struct *vma) +{ + int minor, major; + + minor = iminor(file_inode(vma->vm_file)); + major = imajor(file_inode(vma->vm_file)); + + if ((major == 1) && (minor == 15)) + return true; + else + return false; +} + +int remap_section_range(struct vm_area_struct *vma, unsigned long addr, + unsigned long pfn, unsigned long size, pgprot_t prot) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long end; + pte_t *ptep; + + if (addr & 0x001fffff || pfn & 0x000001ff || size & 0x001fffff) + return -EINVAL; + + if ((pgprot_val(prot) & (PROT_READ | PROT_WRITE)) == PROT_WRITE) + return -EINVAL; + + vm_flags_init(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_HUGETLB); + end = addr + size; + + flush_cache_range(vma, addr, end); + + do { + pte_t entry; + ptep = huge_pte_offset(mm, addr, HUGE_PAGE_SZ_2M); + if (ptep) { + pr_debug("remap_section_range2: %p, %lx\n", ptep, addr); + } else { + ptep = huge_pte_alloc(mm, vma, addr, HUGE_PAGE_SZ_2M); + if (!ptep) { + pr_info("huge_pte_alloc failed\n"); + return -ENOMEM; + } + pr_debug("remap_section_range1: %p, %lx\n", ptep, addr); + } + + entry = pfn_pte(pfn, vma->vm_page_prot); + entry = huge_pte_mkdirty(entry); + entry = huge_pte_mkwrite(entry); + entry = pte_mkyoung(entry); + entry = pte_mkhuge(entry); + set_huge_pte_at(mm, addr, ptep, entry, HUGE_PAGE_SZ_2M); + + pr_debug("remap_section_range: %p, %lx, %llx\n", ptep, addr, entry); + pfn += SZ_2M >> PAGE_SHIFT; + addr += PMD_SIZE; + } while (addr < end); + + return 0; +} + +int arch_mmap_mem_2M(struct file *file, struct vm_area_struct *vma) +{ + size_t size = vma->vm_end - vma->vm_start; + + pr_debug("arch_mmap_mem_2M %.8lx %.8lx to %.8lx\n", + (unsigned long)size, vma->vm_pgoff, vma->vm_start); + + if (!valid_mmap_phys_addr_range(vma->vm_pgoff, size)) + return -EINVAL; + + if (!private_mapping_ok(vma)) + return -ENOSYS; + + if (!range_is_allowed(vma->vm_pgoff, size)) + return -EPERM; + + if (!phys_mem_access_prot_allowed(file, vma->vm_pgoff, size, + &vma->vm_page_prot)) + return -EINVAL; +#if 0 + vma->vm_page_prot = phys_mem_access_prot(file, vma->vm_pgoff, + size, + vma->vm_page_prot); +#endif + return remap_section_range(vma, vma->vm_start, + vma->vm_pgoff, size, + vma->vm_page_prot); +} +#endif + static ssize_t read_port(struct file *file, char __user *buf, size_t count, loff_t *ppos) { @@ -642,6 +740,20 @@ static int open_port(struct inode *inode, struct file *filp) #define write_iter_zero write_iter_null #define open_mem open_port +#ifdef CONFIG_MMAP_HUGEPAGE +static const struct file_operations huge_mem_fops_2M = { + .llseek = memory_lseek, + .read = read_mem, + .write = write_mem, + .mmap = arch_mmap_mem_2M, + .open = open_mem, +#ifndef CONFIG_MMU + .get_unmapped_area = get_unmapped_area_mem, + .mmap_capabilities = memory_mmap_capabilities, +#endif +}; +#endif + static const struct file_operations __maybe_unused mem_fops = { .llseek = memory_lseek, .read = read_mem, @@ -710,6 +822,9 @@ static const struct memdev { #ifdef CONFIG_PRINTK [11] = { "kmsg", &kmsg_fops, 0, 0644 }, #endif +#ifdef CONFIG_MMAP_HUGEPAGE + [15] = { "hugemem2M", &huge_mem_fops_2M, FMODE_UNSIGNED_OFFSET, 0 }, +#endif }; static int memory_open(struct inode *inode, struct file *filp) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 31b2927ada73fa34e877ef1350eb74e8352c0d3a..da0b62ebdbe59e3cf357e34dc2f8944a629e51d1 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -800,8 +800,21 @@ static inline struct hstate *hstate_sizelog(int page_size_log) return NULL; } +#ifdef CONFIG_MMAP_HUGEPAGE +extern bool is_mmap_hp_range(struct vm_area_struct *vma); +#else +static inline bool is_mmap_hp_range(struct vm_area_struct *vma) +{ + return false; +} +#endif + static inline struct hstate *hstate_vma(struct vm_area_struct *vma) { +#ifdef CONFIG_MMAP_HUGEPAGE + if (is_mmap_hp_range(vma)) + return &default_hstate; +#endif return hstate_file(vma->vm_file); } diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 5e6c4d367d33a2a6c61ada21e9bb6af1ffae601e..50a362b64c9769e0dbe1ea11735f73d3e1a2a572 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5133,7 +5133,13 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, make_pte_marker(marker), sz); } else { entry = huge_ptep_get(src_pte); +#ifdef CONFIG_MMAP_HUGEPAGE + if (!is_mmap_hp_range(src_vma)) +#endif pte_folio = page_folio(pte_page(entry)); +#ifdef CONFIG_MMAP_HUGEPAGE + if (!is_mmap_hp_range(src_vma)) +#endif folio_get(pte_folio); /* @@ -5147,6 +5153,9 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, * sleep during the process. */ if (!folio_test_anon(pte_folio)) { +#ifdef CONFIG_MMAP_HUGEPAGE + if (!is_mmap_hp_range(src_vma)) +#endif page_dup_file_rmap(&pte_folio->page, true); } else if (page_try_dup_anon_rmap(&pte_folio->page, true, src_vma)) { @@ -5206,6 +5215,9 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, entry = huge_pte_clear_uffd_wp(entry); set_huge_pte_at(dst, addr, dst_pte, entry, sz); +#ifdef CONFIG_MMAP_HUGEPAGE + if (!is_mmap_hp_range(src_vma)) +#endif hugetlb_count_add(npages, dst); } spin_unlock(src_ptl); @@ -5331,7 +5343,11 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, WARN_ON(!is_vm_hugetlb_page(vma)); BUG_ON(start & ~huge_page_mask(h)); BUG_ON(end & ~huge_page_mask(h)); - +#ifdef CONFIG_MMAP_HUGEPAGE + if (is_mmap_hp_range(vma)) + pr_warn("PID: %d Comm: %s with hp is doing hugepage unmap or exiting!\n", + current->pid, current->comm); +#endif /* * This is a hugetlb vma, all the pte entries should point * to huge page. @@ -5384,7 +5400,9 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, spin_unlock(ptl); continue; } - +#ifdef CONFIG_MMAP_HUGEPAGE + if (!is_mmap_hp_range(vma)) { +#endif page = pte_page(pte); /* * If a reference page is supplied, it is because a specific @@ -5403,9 +5421,14 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, */ set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED); } - +#ifdef CONFIG_MMAP_HUGEPAGE + } +#endif pte = huge_ptep_get_and_clear(mm, address, ptep); tlb_remove_huge_tlb_entry(h, tlb, ptep, address); +#ifdef CONFIG_MMAP_HUGEPAGE + if (!is_mmap_hp_range(vma)) { +#endif if (huge_pte_dirty(pte)) set_page_dirty(page); /* Leave a uffd-wp pte marker if needed */ @@ -5416,8 +5439,13 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, sz); hugetlb_count_sub(pages_per_huge_page(h), mm); page_remove_rmap(page, vma, true); - +#ifdef CONFIG_MMAP_HUGEPAGE + } +#endif spin_unlock(ptl); +#ifdef CONFIG_MMAP_HUGEPAGE + if (!is_mmap_hp_range(vma)) +#endif tlb_remove_page_size(tlb, page, huge_page_size(h)); /* * Bail out after unmapping reference page if supplied @@ -5763,9 +5791,15 @@ static bool hugetlbfs_pagecache_present(struct hstate *h, struct folio *folio; folio = filemap_get_folio(mapping, idx); +#ifdef CONFIG_MMAP_HUGEPAGE + if (!(is_mmap_hp_range(vma))) { +#endif if (IS_ERR(folio)) return false; folio_put(folio); +#ifdef CONFIG_MMAP_HUGEPAGE + } +#endif return true; } diff --git a/nos/extend_features/hugemem-2m/Kconfig b/nos/extend_features/hugemem-2m/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..012c5386ad87df20d473400dd9d8f6ff40a79d9a --- /dev/null +++ b/nos/extend_features/hugemem-2m/Kconfig @@ -0,0 +1,7 @@ +config MMAP_HUGEPAGE + bool "dev/hugemem2M support hugetlb" + depends on HUGETLBFS && !STRICT_DEVMEM + default n + help + dev/hugemem2M support hugetlb +