1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
|
#ifndef _X86_64_PGALLOC_H
#define _X86_64_PGALLOC_H
#include <asm/pda.h>
#include <linux/threads.h>
#include <linux/mm.h>
#include <linux/quicklist.h>
#define QUICK_PGD 0 /* We preserve special mappings over free */
#define QUICK_PT 1 /* Other page table pages that are zero on free */
#define pmd_populate_kernel(mm, pmd, pte) \
set_pmd(pmd, __pmd(_PAGE_TABLE | __pa(pte)))
#define pud_populate(mm, pud, pmd) \
set_pud(pud, __pud(_PAGE_TABLE | __pa(pmd)))
#define pgd_populate(mm, pgd, pud) \
set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(pud)))
static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *pte)
{
set_pmd(pmd, __pmd(_PAGE_TABLE | (page_to_pfn(pte) << PAGE_SHIFT)));
}
static inline void pmd_free(pmd_t *pmd)
{
BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
quicklist_free(QUICK_PT, NULL, pmd);
}
static inline pmd_t *pmd_alloc_one (struct mm_struct *mm, unsigned long addr)
{
return (pmd_t *)quicklist_alloc(QUICK_PT, GFP_KERNEL|__GFP_REPEAT, NULL);
}
static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
{
return (pud_t *)quicklist_alloc(QUICK_PT, GFP_KERNEL|__GFP_REPEAT, NULL);
}
static inline void pud_free (pud_t *pud)
{
BUG_ON((unsigned long)pud & (PAGE_SIZE-1));
quicklist_free(QUICK_PT, NULL, pud);
}
static inline void pgd_list_add(pgd_t *pgd)
{
struct page *page = virt_to_page(pgd);
spin_lock(&pgd_lock);
list_add(&page->lru, &pgd_list);
spin_unlock(&pgd_lock);
}
static inline void pgd_list_del(pgd_t *pgd)
{
struct page *page = virt_to_page(pgd);
spin_lock(&pgd_lock);
list_del(&page->lru);
spin_unlock(&pgd_lock);
}
static inline void pgd_ctor(void *x)
{
unsigned boundary;
pgd_t *pgd = x;
struct page *page = virt_to_page(pgd);
/*
* Copy kernel pointers in from init.
*/
boundary = pgd_index(__PAGE_OFFSET);
memcpy(pgd + boundary,
init_level4_pgt + boundary,
(PTRS_PER_PGD - boundary) * sizeof(pgd_t));
spin_lock(&pgd_lock);
list_add(&page->lru, &pgd_list);
spin_unlock(&pgd_lock);
}
static inline void pgd_dtor(void *x)
{
pgd_t *pgd = x;
struct page *page = virt_to_page(pgd);
spin_lock(&pgd_lock);
list_del(&page->lru);
spin_unlock(&pgd_lock);
}
static inline pgd_t *pgd_alloc(struct mm_struct *mm)
{
pgd_t *pgd = (pgd_t *)quicklist_alloc(QUICK_PGD,
GFP_KERNEL|__GFP_REPEAT, pgd_ctor);
return pgd;
}
static inline void pgd_free(pgd_t *pgd)
{
BUG_ON((unsigned long)pgd & (PAGE_SIZE-1));
quicklist_free(QUICK_PGD, pgd_dtor, pgd);
}
static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
{
return (pte_t *)quicklist_alloc(QUICK_PT, GFP_KERNEL|__GFP_REPEAT, NULL);
}
static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
void *p = (void *)quicklist_alloc(QUICK_PT, GFP_KERNEL|__GFP_REPEAT, NULL);
if (!p)
return NULL;
return virt_to_page(p);
}
/* Should really implement gc for free page table pages. This could be
done with a reference count in struct page. */
static inline void pte_free_kernel(pte_t *pte)
{
BUG_ON((unsigned long)pte & (PAGE_SIZE-1));
quicklist_free(QUICK_PT, NULL, pte);
}
static inline void pte_free(struct page *pte)
{
quicklist_free_page(QUICK_PT, NULL, pte);
}
#define __pte_free_tlb(tlb,pte) quicklist_free_page(QUICK_PT, NULL,(pte))
#define __pmd_free_tlb(tlb,x) quicklist_free(QUICK_PT, NULL, (x))
#define __pud_free_tlb(tlb,x) quicklist_free(QUICK_PT, NULL, (x))
static inline void check_pgt_cache(void)
{
quicklist_trim(QUICK_PGD, pgd_dtor, 25, 16);
quicklist_trim(QUICK_PT, NULL, 25, 16);
}
#endif /* _X86_64_PGALLOC_H */
|
