summaryrefslogtreecommitdiffstats
path: root/mm
diff options
context:
space:
mode:
Diffstat (limited to 'mm')
-rw-r--r--mm/hugetlb.c3
-rw-r--r--mm/memblock.c7
-rw-r--r--mm/memcontrol.c24
-rw-r--r--mm/memory.c28
-rw-r--r--mm/mempolicy.c11
-rw-r--r--mm/migrate.c16
-rw-r--r--mm/mmap.c77
-rw-r--r--mm/nobootmem.c13
-rw-r--r--mm/nommu.c41
-rw-r--r--mm/page_alloc.c2
-rw-r--r--mm/percpu.c22
-rw-r--r--mm/slub.c2
-rw-r--r--mm/swap_state.c2
-rw-r--r--mm/vmscan.c11
-rw-r--r--mm/vmstat.c4
15 files changed, 182 insertions, 81 deletions
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index cd65cb19c94..ae8f708e3d7 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -532,7 +532,7 @@ static struct page *dequeue_huge_page_vma(struct hstate *h,
struct vm_area_struct *vma,
unsigned long address, int avoid_reserve)
{
- struct page *page;
+ struct page *page = NULL;
struct mempolicy *mpol;
nodemask_t *nodemask;
struct zonelist *zonelist;
@@ -2498,7 +2498,6 @@ retry_avoidcopy:
if (outside_reserve) {
BUG_ON(huge_pte_none(pte));
if (unmap_ref_private(mm, vma, old_page, address)) {
- BUG_ON(page_count(old_page) != 1);
BUG_ON(huge_pte_none(pte));
spin_lock(&mm->page_table_lock);
ptep = huge_pte_offset(mm, address & huge_page_mask(h));
diff --git a/mm/memblock.c b/mm/memblock.c
index 99f28559950..a44eab3157f 100644
--- a/mm/memblock.c
+++ b/mm/memblock.c
@@ -330,6 +330,9 @@ static int __init_memblock memblock_add_region(struct memblock_type *type,
phys_addr_t end = base + memblock_cap_size(base, &size);
int i, nr_new;
+ if (!size)
+ return 0;
+
/* special case for empty array */
if (type->regions[0].size == 0) {
WARN_ON(type->cnt != 1 || type->total_size);
@@ -430,6 +433,9 @@ static int __init_memblock memblock_isolate_range(struct memblock_type *type,
*start_rgn = *end_rgn = 0;
+ if (!size)
+ return 0;
+
/* we'll create at most two more regions */
while (type->cnt + 2 > type->max)
if (memblock_double_array(type) < 0)
@@ -514,7 +520,6 @@ int __init_memblock memblock_reserve(phys_addr_t base, phys_addr_t size)
(unsigned long long)base,
(unsigned long long)base + size,
(void *)_RET_IP_);
- BUG_ON(0 == size);
return memblock_add_region(_rgn, base, size, MAX_NUMNODES);
}
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index a7165a60d0a..7685d4a0b3c 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2476,10 +2476,10 @@ struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
static void __mem_cgroup_commit_charge(struct mem_cgroup *memcg,
struct page *page,
unsigned int nr_pages,
- struct page_cgroup *pc,
enum charge_type ctype,
bool lrucare)
{
+ struct page_cgroup *pc = lookup_page_cgroup(page);
struct zone *uninitialized_var(zone);
bool was_on_lru = false;
bool anon;
@@ -2716,7 +2716,6 @@ static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
{
struct mem_cgroup *memcg = NULL;
unsigned int nr_pages = 1;
- struct page_cgroup *pc;
bool oom = true;
int ret;
@@ -2730,11 +2729,10 @@ static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
oom = false;
}
- pc = lookup_page_cgroup(page);
ret = __mem_cgroup_try_charge(mm, gfp_mask, nr_pages, &memcg, oom);
if (ret == -ENOMEM)
return ret;
- __mem_cgroup_commit_charge(memcg, page, nr_pages, pc, ctype, false);
+ __mem_cgroup_commit_charge(memcg, page, nr_pages, ctype, false);
return 0;
}
@@ -2831,16 +2829,13 @@ static void
__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *memcg,
enum charge_type ctype)
{
- struct page_cgroup *pc;
-
if (mem_cgroup_disabled())
return;
if (!memcg)
return;
cgroup_exclude_rmdir(&memcg->css);
- pc = lookup_page_cgroup(page);
- __mem_cgroup_commit_charge(memcg, page, 1, pc, ctype, true);
+ __mem_cgroup_commit_charge(memcg, page, 1, ctype, true);
/*
* Now swap is on-memory. This means this page may be
* counted both as mem and swap....double count.
@@ -3298,14 +3293,13 @@ int mem_cgroup_prepare_migration(struct page *page,
* page. In the case new page is migrated but not remapped, new page's
* mapcount will be finally 0 and we call uncharge in end_migration().
*/
- pc = lookup_page_cgroup(newpage);
if (PageAnon(page))
ctype = MEM_CGROUP_CHARGE_TYPE_MAPPED;
else if (page_is_file_cache(page))
ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
else
ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM;
- __mem_cgroup_commit_charge(memcg, newpage, 1, pc, ctype, false);
+ __mem_cgroup_commit_charge(memcg, newpage, 1, ctype, false);
return ret;
}
@@ -3392,7 +3386,7 @@ void mem_cgroup_replace_page_cache(struct page *oldpage,
* the newpage may be on LRU(or pagevec for LRU) already. We lock
* LRU while we overwrite pc->mem_cgroup.
*/
- __mem_cgroup_commit_charge(memcg, newpage, 1, pc, type, true);
+ __mem_cgroup_commit_charge(memcg, newpage, 1, type, true);
}
#ifdef CONFIG_DEBUG_VM
@@ -4513,6 +4507,12 @@ static void mem_cgroup_usage_unregister_event(struct cgroup *cgrp,
swap_buffers:
/* Swap primary and spare array */
thresholds->spare = thresholds->primary;
+ /* If all events are unregistered, free the spare array */
+ if (!new) {
+ kfree(thresholds->spare);
+ thresholds->spare = NULL;
+ }
+
rcu_assign_pointer(thresholds->primary, new);
/* To be sure that nobody uses thresholds */
@@ -5481,7 +5481,7 @@ static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
* part of thp split is not executed yet.
*/
if (pmd_trans_huge_lock(pmd, vma) == 1) {
- if (!mc.precharge) {
+ if (mc.precharge < HPAGE_PMD_NR) {
spin_unlock(&vma->vm_mm->page_table_lock);
return 0;
}
diff --git a/mm/memory.c b/mm/memory.c
index 6105f475fa8..1e77da6d82c 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1295,7 +1295,7 @@ static void unmap_page_range(struct mmu_gather *tlb,
static void unmap_single_vma(struct mmu_gather *tlb,
struct vm_area_struct *vma, unsigned long start_addr,
- unsigned long end_addr, unsigned long *nr_accounted,
+ unsigned long end_addr,
struct zap_details *details)
{
unsigned long start = max(vma->vm_start, start_addr);
@@ -1307,9 +1307,6 @@ static void unmap_single_vma(struct mmu_gather *tlb,
if (end <= vma->vm_start)
return;
- if (vma->vm_flags & VM_ACCOUNT)
- *nr_accounted += (end - start) >> PAGE_SHIFT;
-
if (unlikely(is_pfn_mapping(vma)))
untrack_pfn_vma(vma, 0, 0);
@@ -1339,8 +1336,6 @@ static void unmap_single_vma(struct mmu_gather *tlb,
* @vma: the starting vma
* @start_addr: virtual address at which to start unmapping
* @end_addr: virtual address at which to end unmapping
- * @nr_accounted: Place number of unmapped pages in vm-accountable vma's here
- * @details: details of nonlinear truncation or shared cache invalidation
*
* Unmap all pages in the vma list.
*
@@ -1355,15 +1350,13 @@ static void unmap_single_vma(struct mmu_gather *tlb,
*/
void unmap_vmas(struct mmu_gather *tlb,
struct vm_area_struct *vma, unsigned long start_addr,
- unsigned long end_addr, unsigned long *nr_accounted,
- struct zap_details *details)
+ unsigned long end_addr)
{
struct mm_struct *mm = vma->vm_mm;
mmu_notifier_invalidate_range_start(mm, start_addr, end_addr);
for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next)
- unmap_single_vma(tlb, vma, start_addr, end_addr, nr_accounted,
- details);
+ unmap_single_vma(tlb, vma, start_addr, end_addr, NULL);
mmu_notifier_invalidate_range_end(mm, start_addr, end_addr);
}
@@ -1376,19 +1369,21 @@ void unmap_vmas(struct mmu_gather *tlb,
*
* Caller must protect the VMA list
*/
-void zap_page_range(struct vm_area_struct *vma, unsigned long address,
+void zap_page_range(struct vm_area_struct *vma, unsigned long start,
unsigned long size, struct zap_details *details)
{
struct mm_struct *mm = vma->vm_mm;
struct mmu_gather tlb;
- unsigned long end = address + size;
- unsigned long nr_accounted = 0;
+ unsigned long end = start + size;
lru_add_drain();
tlb_gather_mmu(&tlb, mm, 0);
update_hiwater_rss(mm);
- unmap_vmas(&tlb, vma, address, end, &nr_accounted, details);
- tlb_finish_mmu(&tlb, address, end);
+ mmu_notifier_invalidate_range_start(mm, start, end);
+ for ( ; vma && vma->vm_start < end; vma = vma->vm_next)
+ unmap_single_vma(&tlb, vma, start, end, details);
+ mmu_notifier_invalidate_range_end(mm, start, end);
+ tlb_finish_mmu(&tlb, start, end);
}
/**
@@ -1406,13 +1401,12 @@ static void zap_page_range_single(struct vm_area_struct *vma, unsigned long addr
struct mm_struct *mm = vma->vm_mm;
struct mmu_gather tlb;
unsigned long end = address + size;
- unsigned long nr_accounted = 0;
lru_add_drain();
tlb_gather_mmu(&tlb, mm, 0);
update_hiwater_rss(mm);
mmu_notifier_invalidate_range_start(mm, address, end);
- unmap_single_vma(&tlb, vma, address, end, &nr_accounted, details);
+ unmap_single_vma(&tlb, vma, address, end, details);
mmu_notifier_invalidate_range_end(mm, address, end);
tlb_finish_mmu(&tlb, address, end);
}
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index cfb6c867875..b1956913752 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -1361,11 +1361,14 @@ SYSCALL_DEFINE4(migrate_pages, pid_t, pid, unsigned long, maxnode,
mm = get_task_mm(task);
put_task_struct(task);
- if (mm)
- err = do_migrate_pages(mm, old, new,
- capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);
- else
+
+ if (!mm) {
err = -EINVAL;
+ goto out;
+ }
+
+ err = do_migrate_pages(mm, old, new,
+ capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);
mmput(mm);
out:
diff --git a/mm/migrate.c b/mm/migrate.c
index 51c08a0c6f6..11072383ae1 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -1388,14 +1388,14 @@ SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages,
mm = get_task_mm(task);
put_task_struct(task);
- if (mm) {
- if (nodes)
- err = do_pages_move(mm, task_nodes, nr_pages, pages,
- nodes, status, flags);
- else
- err = do_pages_stat(mm, nr_pages, pages, status);
- } else
- err = -EINVAL;
+ if (!mm)
+ return -EINVAL;
+
+ if (nodes)
+ err = do_pages_move(mm, task_nodes, nr_pages, pages,
+ nodes, status, flags);
+ else
+ err = do_pages_stat(mm, nr_pages, pages, status);
mmput(mm);
return err;
diff --git a/mm/mmap.c b/mm/mmap.c
index a7bf6a31c9f..69a1889f379 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -240,6 +240,8 @@ static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
return next;
}
+static unsigned long do_brk(unsigned long addr, unsigned long len);
+
SYSCALL_DEFINE1(brk, unsigned long, brk)
{
unsigned long rlim, retval;
@@ -951,7 +953,7 @@ static inline unsigned long round_hint_to_min(unsigned long hint)
* The caller must hold down_write(&current->mm->mmap_sem).
*/
-unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
+static unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
unsigned long len, unsigned long prot,
unsigned long flags, unsigned long pgoff)
{
@@ -1087,7 +1089,32 @@ unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
return mmap_region(file, addr, len, flags, vm_flags, pgoff);
}
-EXPORT_SYMBOL(do_mmap_pgoff);
+
+unsigned long do_mmap(struct file *file, unsigned long addr,
+ unsigned long len, unsigned long prot,
+ unsigned long flag, unsigned long offset)
+{
+ if (unlikely(offset + PAGE_ALIGN(len) < offset))
+ return -EINVAL;
+ if (unlikely(offset & ~PAGE_MASK))
+ return -EINVAL;
+ return do_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT);
+}
+EXPORT_SYMBOL(do_mmap);
+
+unsigned long vm_mmap(struct file *file, unsigned long addr,
+ unsigned long len, unsigned long prot,
+ unsigned long flag, unsigned long offset)
+{
+ unsigned long ret;
+ struct mm_struct *mm = current->mm;
+
+ down_write(&mm->mmap_sem);
+ ret = do_mmap(file, addr, len, prot, flag, offset);
+ up_write(&mm->mmap_sem);
+ return ret;
+}
+EXPORT_SYMBOL(vm_mmap);
SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
unsigned long, prot, unsigned long, flags,
@@ -1862,15 +1889,20 @@ find_extend_vma(struct mm_struct * mm, unsigned long addr)
*/
static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma)
{
+ unsigned long nr_accounted = 0;
+
/* Update high watermark before we lower total_vm */
update_hiwater_vm(mm);
do {
long nrpages = vma_pages(vma);
+ if (vma->vm_flags & VM_ACCOUNT)
+ nr_accounted += nrpages;
mm->total_vm -= nrpages;
vm_stat_account(mm, vma->vm_flags, vma->vm_file, -nrpages);
vma = remove_vma(vma);
} while (vma);
+ vm_unacct_memory(nr_accounted);
validate_mm(mm);
}
@@ -1885,13 +1917,11 @@ static void unmap_region(struct mm_struct *mm,
{
struct vm_area_struct *next = prev? prev->vm_next: mm->mmap;
struct mmu_gather tlb;
- unsigned long nr_accounted = 0;
lru_add_drain();
tlb_gather_mmu(&tlb, mm, 0);
update_hiwater_rss(mm);
- unmap_vmas(&tlb, vma, start, end, &nr_accounted, NULL);
- vm_unacct_memory(nr_accounted);
+ unmap_vmas(&tlb, vma, start, end);
free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS,
next ? next->vm_start : 0);
tlb_finish_mmu(&tlb, start, end);
@@ -2105,21 +2135,25 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
return 0;
}
-
EXPORT_SYMBOL(do_munmap);
-SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
+int vm_munmap(unsigned long start, size_t len)
{
int ret;
struct mm_struct *mm = current->mm;
- profile_munmap(addr);
-
down_write(&mm->mmap_sem);
- ret = do_munmap(mm, addr, len);
+ ret = do_munmap(mm, start, len);
up_write(&mm->mmap_sem);
return ret;
}
+EXPORT_SYMBOL(vm_munmap);
+
+SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
+{
+ profile_munmap(addr);
+ return vm_munmap(addr, len);
+}
static inline void verify_mm_writelocked(struct mm_struct *mm)
{
@@ -2136,7 +2170,7 @@ static inline void verify_mm_writelocked(struct mm_struct *mm)
* anonymous maps. eventually we may be able to do some
* brk-specific accounting here.
*/
-unsigned long do_brk(unsigned long addr, unsigned long len)
+static unsigned long do_brk(unsigned long addr, unsigned long len)
{
struct mm_struct * mm = current->mm;
struct vm_area_struct * vma, * prev;
@@ -2232,7 +2266,17 @@ out:
return addr;
}
-EXPORT_SYMBOL(do_brk);
+unsigned long vm_brk(unsigned long addr, unsigned long len)
+{
+ struct mm_struct *mm = current->mm;
+ unsigned long ret;
+
+ down_write(&mm->mmap_sem);
+ ret = do_brk(addr, len);
+ up_write(&mm->mmap_sem);
+ return ret;
+}
+EXPORT_SYMBOL(vm_brk);
/* Release all mmaps. */
void exit_mmap(struct mm_struct *mm)
@@ -2264,8 +2308,7 @@ void exit_mmap(struct mm_struct *mm)
tlb_gather_mmu(&tlb, mm, 1);
/* update_hiwater_rss(mm) here? but nobody should be looking */
/* Use -1 here to ensure all VMAs in the mm are unmapped */
- unmap_vmas(&tlb, vma, 0, -1, &nr_accounted, NULL);
- vm_unacct_memory(nr_accounted);
+ unmap_vmas(&tlb, vma, 0, -1);
free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, 0);
tlb_finish_mmu(&tlb, 0, -1);
@@ -2274,8 +2317,12 @@ void exit_mmap(struct mm_struct *mm)
* Walk the list again, actually closing and freeing it,
* with preemption enabled, without holding any MM locks.
*/
- while (vma)
+ while (vma) {
+ if (vma->vm_flags & VM_ACCOUNT)
+ nr_accounted += vma_pages(vma);
vma = remove_vma(vma);
+ }
+ vm_unacct_memory(nr_accounted);
BUG_ON(mm->nr_ptes > (FIRST_USER_ADDRESS+PMD_SIZE-1)>>PMD_SHIFT);
}
diff --git a/mm/nobootmem.c b/mm/nobootmem.c
index 24f0fc1a56d..1983fb1c702 100644
--- a/mm/nobootmem.c
+++ b/mm/nobootmem.c
@@ -82,8 +82,7 @@ void __init free_bootmem_late(unsigned long addr, unsigned long size)
static void __init __free_pages_memory(unsigned long start, unsigned long end)
{
- int i;
- unsigned long start_aligned, end_aligned;
+ unsigned long i, start_aligned, end_aligned;
int order = ilog2(BITS_PER_LONG);
start_aligned = (start + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1);
@@ -298,13 +297,19 @@ void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
if (WARN_ON_ONCE(slab_is_available()))
return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
+again:
ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
goal, -1ULL);
if (ptr)
return ptr;
- return __alloc_memory_core_early(MAX_NUMNODES, size, align,
- goal, -1ULL);
+ ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align,
+ goal, -1ULL);
+ if (!ptr && goal) {
+ goal = 0;
+ goto again;
+ }
+ return ptr;
}
void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size,
diff --git a/mm/nommu.c b/mm/nommu.c
index f59e170fceb..bb8f4f004a8 100644
--- a/mm/nommu.c
+++ b/mm/nommu.c
@@ -1233,7 +1233,7 @@ enomem:
/*
* handle mapping creation for uClinux
*/
-unsigned long do_mmap_pgoff(struct file *file,
+static unsigned long do_mmap_pgoff(struct file *file,
unsigned long addr,
unsigned long len,
unsigned long prot,
@@ -1470,7 +1470,32 @@ error_getting_region:
show_free_areas(0);
return -ENOMEM;
}
-EXPORT_SYMBOL(do_mmap_pgoff);
+
+unsigned long do_mmap(struct file *file, unsigned long addr,
+ unsigned long len, unsigned long prot,
+ unsigned long flag, unsigned long offset)
+{
+ if (unlikely(offset + PAGE_ALIGN(len) < offset))
+ return -EINVAL;
+ if (unlikely(offset & ~PAGE_MASK))
+ return -EINVAL;
+ return do_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT);
+}
+EXPORT_SYMBOL(do_mmap);
+
+unsigned long vm_mmap(struct file *file, unsigned long addr,
+ unsigned long len, unsigned long prot,
+ unsigned long flag, unsigned long offset)
+{
+ unsigned long ret;
+ struct mm_struct *mm = current->mm;
+
+ down_write(&mm->mmap_sem);
+ ret = do_mmap(file, addr, len, prot, flag, offset);
+ up_write(&mm->mmap_sem);
+ return ret;
+}
+EXPORT_SYMBOL(vm_mmap);
SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
unsigned long, prot, unsigned long, flags,
@@ -1709,16 +1734,22 @@ erase_whole_vma:
}
EXPORT_SYMBOL(do_munmap);
-SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
+int vm_munmap(unsigned long addr, size_t len)
{
- int ret;
struct mm_struct *mm = current->mm;
+ int ret;
down_write(&mm->mmap_sem);
ret = do_munmap(mm, addr, len);
up_write(&mm->mmap_sem);
return ret;
}
+EXPORT_SYMBOL(vm_munmap);
+
+SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
+{
+ return vm_munmap(addr, len);
+}
/*
* release all the mappings made in a process's VM space
@@ -1744,7 +1775,7 @@ void exit_mmap(struct mm_struct *mm)
kleave("");
}
-unsigned long do_brk(unsigned long addr, unsigned long len)
+unsigned long vm_brk(unsigned long addr, unsigned long len)
{
return -ENOMEM;
}
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index a712fb9e04c..918330f71db 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -5203,7 +5203,7 @@ int percpu_pagelist_fraction_sysctl_handler(ctl_table *table, int write,
int ret;
ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
- if (!write || (ret == -EINVAL))
+ if (!write || (ret < 0))
return ret;
for_each_populated_zone(zone) {
for_each_possible_cpu(cpu) {
diff --git a/mm/percpu.c b/mm/percpu.c
index f47af9123af..bb4be7435ce 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -1132,20 +1132,20 @@ static void pcpu_dump_alloc_info(const char *lvl,
for (alloc_end += gi->nr_units / upa;
alloc < alloc_end; alloc++) {
if (!(alloc % apl)) {
- printk("\n");
+ printk(KERN_CONT "\n");
printk("%spcpu-alloc: ", lvl);
}
- printk("[%0*d] ", group_width, group);
+ printk(KERN_CONT "[%0*d] ", group_width, group);
for (unit_end += upa; unit < unit_end; unit++)
if (gi->cpu_map[unit] != NR_CPUS)
- printk("%0*d ", cpu_width,
+ printk(KERN_CONT "%0*d ", cpu_width,
gi->cpu_map[unit]);
else
- printk("%s ", empty_str);
+ printk(KERN_CONT "%s ", empty_str);
}
}
- printk("\n");
+ printk(KERN_CONT "\n");
}
/**
@@ -1650,6 +1650,16 @@ int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size,
areas[group] = ptr;
base = min(ptr, base);
+ }
+
+ /*
+ * Copy data and free unused parts. This should happen after all
+ * allocations are complete; otherwise, we may end up with
+ * overlapping groups.
+ */
+ for (group = 0; group < ai->nr_groups; group++) {
+ struct pcpu_group_info *gi = &ai->groups[group];
+ void *ptr = areas[group];
for (i = 0; i < gi->nr_units; i++, ptr += ai->unit_size) {
if (gi->cpu_map[i] == NR_CPUS) {
@@ -1885,6 +1895,8 @@ void __init setup_per_cpu_areas(void)
fc = __alloc_bootmem(unit_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
if (!ai || !fc)
panic("Failed to allocate memory for percpu areas.");
+ /* kmemleak tracks the percpu allocations separately */
+ kmemleak_free(fc);
ai->dyn_size = unit_size;
ai->unit_size = unit_size;
diff --git a/mm/slub.c b/mm/slub.c
index ffe13fdf814..80848cd3901 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -2040,7 +2040,7 @@ static bool has_cpu_slab(int cpu, void *info)
struct kmem_cache *s = info;
struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
- return !!(c->page);
+ return c->page || c->partial;
}
static void flush_all(struct kmem_cache *s)
diff --git a/mm/swap_state.c b/mm/swap_state.c
index 9d3dd3763cf..4c5ff7f284d 100644
--- a/mm/swap_state.c
+++ b/mm/swap_state.c
@@ -26,7 +26,7 @@
*/
static const struct address_space_operations swap_aops = {
.writepage = swap_writepage,
- .set_page_dirty = __set_page_dirty_nobuffers,
+ .set_page_dirty = __set_page_dirty_no_writeback,
.migratepage = migrate_page,
};
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 1a518684a32..33dc256033b 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1568,9 +1568,14 @@ shrink_inactive_list(unsigned long nr_to_scan, struct mem_cgroup_zone *mz,
reclaim_stat->recent_scanned[0] += nr_anon;
reclaim_stat->recent_scanned[1] += nr_file;
- if (current_is_kswapd())
- __count_vm_events(KSWAPD_STEAL, nr_reclaimed);
- __count_zone_vm_events(PGSTEAL, zone, nr_reclaimed);
+ if (global_reclaim(sc)) {
+ if (current_is_kswapd())
+ __count_zone_vm_events(PGSTEAL_KSWAPD, zone,
+ nr_reclaimed);
+ else
+ __count_zone_vm_events(PGSTEAL_DIRECT, zone,
+ nr_reclaimed);
+ }
putback_inactive_pages(mz, &page_list);
diff --git a/mm/vmstat.c b/mm/vmstat.c
index f600557a765..7db1b9bab49 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -738,7 +738,8 @@ const char * const vmstat_text[] = {
"pgmajfault",
TEXTS_FOR_ZONES("pgrefill")
- TEXTS_FOR_ZONES("pgsteal")
+ TEXTS_FOR_ZONES("pgsteal_kswapd")
+ TEXTS_FOR_ZONES("pgsteal_direct")
TEXTS_FOR_ZONES("pgscan_kswapd")
TEXTS_FOR_ZONES("pgscan_direct")
@@ -747,7 +748,6 @@ const char * const vmstat_text[] = {
#endif
"pginodesteal",
"slabs_scanned",
- "kswapd_steal",
"kswapd_inodesteal",
"kswapd_low_wmark_hit_quickly",
"kswapd_high_wmark_hit_quickly",
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2024-11-20 17:17:28 +0000