summaryrefslogtreecommitdiffstats
path: root/arch/powerpc/mm/hugetlbpage.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/powerpc/mm/hugetlbpage.c')
-rw-r--r--arch/powerpc/mm/hugetlbpage.c86
1 files changed, 48 insertions, 38 deletions
diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c
index f1c2d55b437..201c7a5486c 100644
--- a/arch/powerpc/mm/hugetlbpage.c
+++ b/arch/powerpc/mm/hugetlbpage.c
@@ -53,8 +53,7 @@ unsigned int mmu_huge_psizes[MMU_PAGE_COUNT] = { }; /* initialize all to 0 */
/* Subtract one from array size because we don't need a cache for 4K since
* is not a huge page size */
-#define huge_pgtable_cache(psize) (pgtable_cache[HUGEPTE_CACHE_NUM \
- + psize-1])
+#define HUGE_PGTABLE_INDEX(psize) (HUGEPTE_CACHE_NUM + psize - 1)
#define HUGEPTE_CACHE_NAME(psize) (huge_pgtable_cache_name[psize])
static const char *huge_pgtable_cache_name[MMU_PAGE_COUNT] = {
@@ -113,7 +112,7 @@ static inline pte_t *hugepte_offset(hugepd_t *hpdp, unsigned long addr,
static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
unsigned long address, unsigned int psize)
{
- pte_t *new = kmem_cache_zalloc(huge_pgtable_cache(psize),
+ pte_t *new = kmem_cache_zalloc(pgtable_cache[HUGE_PGTABLE_INDEX(psize)],
GFP_KERNEL|__GFP_REPEAT);
if (! new)
@@ -121,36 +120,44 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
spin_lock(&mm->page_table_lock);
if (!hugepd_none(*hpdp))
- kmem_cache_free(huge_pgtable_cache(psize), new);
+ kmem_cache_free(pgtable_cache[HUGE_PGTABLE_INDEX(psize)], new);
else
hpdp->pd = (unsigned long)new | HUGEPD_OK;
spin_unlock(&mm->page_table_lock);
return 0;
}
-/* Base page size affects how we walk hugetlb page tables */
-#ifdef CONFIG_PPC_64K_PAGES
-#define hpmd_offset(pud, addr, h) pmd_offset(pud, addr)
-#define hpmd_alloc(mm, pud, addr, h) pmd_alloc(mm, pud, addr)
-#else
-static inline
-pmd_t *hpmd_offset(pud_t *pud, unsigned long addr, struct hstate *hstate)
+
+static pud_t *hpud_offset(pgd_t *pgd, unsigned long addr, struct hstate *hstate)
+{
+ if (huge_page_shift(hstate) < PUD_SHIFT)
+ return pud_offset(pgd, addr);
+ else
+ return (pud_t *) pgd;
+}
+static pud_t *hpud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long addr,
+ struct hstate *hstate)
{
- if (huge_page_shift(hstate) == PAGE_SHIFT_64K)
+ if (huge_page_shift(hstate) < PUD_SHIFT)
+ return pud_alloc(mm, pgd, addr);
+ else
+ return (pud_t *) pgd;
+}
+static pmd_t *hpmd_offset(pud_t *pud, unsigned long addr, struct hstate *hstate)
+{
+ if (huge_page_shift(hstate) < PMD_SHIFT)
return pmd_offset(pud, addr);
else
return (pmd_t *) pud;
}
-static inline
-pmd_t *hpmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long addr,
- struct hstate *hstate)
+static pmd_t *hpmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long addr,
+ struct hstate *hstate)
{
- if (huge_page_shift(hstate) == PAGE_SHIFT_64K)
+ if (huge_page_shift(hstate) < PMD_SHIFT)
return pmd_alloc(mm, pud, addr);
else
return (pmd_t *) pud;
}
-#endif
/* Build list of addresses of gigantic pages. This function is used in early
* boot before the buddy or bootmem allocator is setup.
@@ -204,7 +211,7 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
pg = pgd_offset(mm, addr);
if (!pgd_none(*pg)) {
- pu = pud_offset(pg, addr);
+ pu = hpud_offset(pg, addr, hstate);
if (!pud_none(*pu)) {
pm = hpmd_offset(pu, addr, hstate);
if (!pmd_none(*pm))
@@ -233,7 +240,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
addr &= hstate->mask;
pg = pgd_offset(mm, addr);
- pu = pud_alloc(mm, pg, addr);
+ pu = hpud_alloc(mm, pg, addr, hstate);
if (pu) {
pm = hpmd_alloc(mm, pu, addr, hstate);
@@ -316,13 +323,7 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
pud = pud_offset(pgd, addr);
do {
next = pud_addr_end(addr, end);
-#ifdef CONFIG_PPC_64K_PAGES
- if (pud_none_or_clear_bad(pud))
- continue;
- hugetlb_free_pmd_range(tlb, pud, addr, next, floor, ceiling,
- psize);
-#else
- if (shift == PAGE_SHIFT_64K) {
+ if (shift < PMD_SHIFT) {
if (pud_none_or_clear_bad(pud))
continue;
hugetlb_free_pmd_range(tlb, pud, addr, next, floor,
@@ -332,7 +333,6 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
continue;
free_hugepte_range(tlb, (hugepd_t *)pud, psize);
}
-#endif
} while (pud++, addr = next, addr != end);
start &= PGDIR_MASK;
@@ -422,9 +422,15 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb,
psize = get_slice_psize(tlb->mm, addr);
BUG_ON(!mmu_huge_psizes[psize]);
next = pgd_addr_end(addr, end);
- if (pgd_none_or_clear_bad(pgd))
- continue;
- hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling);
+ if (mmu_psize_to_shift(psize) < PUD_SHIFT) {
+ if (pgd_none_or_clear_bad(pgd))
+ continue;
+ hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling);
+ } else {
+ if (pgd_none(*pgd))
+ continue;
+ free_hugepte_range(tlb, (hugepd_t *)pgd, psize);
+ }
} while (pgd++, addr = next, addr != end);
}
@@ -500,6 +506,9 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
{
struct hstate *hstate = hstate_file(file);
int mmu_psize = shift_to_mmu_psize(huge_page_shift(hstate));
+
+ if (!mmu_huge_psizes[mmu_psize])
+ return -EINVAL;
return slice_get_unmapped_area(addr, len, flags, mmu_psize, 1, 0);
}
@@ -670,7 +679,7 @@ repeat:
return err;
}
-void set_huge_psize(int psize)
+static void __init set_huge_psize(int psize)
{
/* Check that it is a page size supported by the hardware and
* that it fits within pagetable limits. */
@@ -753,13 +762,14 @@ static int __init hugetlbpage_init(void)
for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
if (mmu_huge_psizes[psize]) {
- huge_pgtable_cache(psize) = kmem_cache_create(
- HUGEPTE_CACHE_NAME(psize),
- HUGEPTE_TABLE_SIZE(psize),
- HUGEPTE_TABLE_SIZE(psize),
- 0,
- NULL);
- if (!huge_pgtable_cache(psize))
+ pgtable_cache[HUGE_PGTABLE_INDEX(psize)] =
+ kmem_cache_create(
+ HUGEPTE_CACHE_NAME(psize),
+ HUGEPTE_TABLE_SIZE(psize),
+ HUGEPTE_TABLE_SIZE(psize),
+ 0,
+ NULL);
+ if (!pgtable_cache[HUGE_PGTABLE_INDEX(psize)])
panic("hugetlbpage_init(): could not create %s"\
"\n", HUGEPTE_CACHE_NAME(psize));
}
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2024-12-26 05:39:48 +0000