From 81fcfb813fe99c30f77dd3ed9a4e541d14a9ed01 Mon Sep 17 00:00:00 2001 From: Alexey Kardashevskiy Date: Wed, 28 Aug 2013 18:37:39 +1000 Subject: hashtable: add hash_for_each_possible_rcu_notrace() This adds hash_for_each_possible_rcu_notrace() which is basically a notrace clone of hash_for_each_possible_rcu() which cannot be used in real mode due to its tracing/debugging capability. Signed-off-by: Alexey Kardashevskiy Signed-off-by: Benjamin Herrenschmidt --- include/linux/hashtable.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'include') diff --git a/include/linux/hashtable.h b/include/linux/hashtable.h index a9df51f5d54..519b6e2d769 100644 --- a/include/linux/hashtable.h +++ b/include/linux/hashtable.h @@ -173,6 +173,21 @@ static inline void hash_del_rcu(struct hlist_node *node) hlist_for_each_entry_rcu(obj, &name[hash_min(key, HASH_BITS(name))],\ member) +/** + * hash_for_each_possible_rcu_notrace - iterate over all possible objects hashing + * to the same bucket in an rcu enabled hashtable in a rcu enabled hashtable + * @name: hashtable to iterate + * @obj: the type * to use as a loop cursor for each entry + * @member: the name of the hlist_node within the struct + * @key: the key of the objects to iterate over + * + * This is the same as hash_for_each_possible_rcu() except that it does + * not do any RCU debugging or tracing. + */ +#define hash_for_each_possible_rcu_notrace(name, obj, member, key) \ + hlist_for_each_entry_rcu_notrace(obj, \ + &name[hash_min(key, HASH_BITS(name))], member) + /** * hash_for_each_possible_safe - iterate over all possible objects hashing to the * same bucket safe against removals -- cgit v1.2.3-70-g09d2 From 8e0861fa3c4edfc2f30dd4cf4d58d3929f7c1b23 Mon Sep 17 00:00:00 2001 From: Alexey Kardashevskiy Date: Wed, 28 Aug 2013 18:37:42 +1000 Subject: powerpc: Prepare to support kernel handling of IOMMU map/unmap The current VFIO-on-POWER implementation supports only user mode driven mapping, i.e. QEMU is sending requests to map/unmap pages. However this approach is really slow, so we want to move that to KVM. Since H_PUT_TCE can be extremely performance sensitive (especially with network adapters where each packet needs to be mapped/unmapped) we chose to implement that as a "fast" hypercall directly in "real mode" (processor still in the guest context but MMU off). To be able to do that, we need to provide some facilities to access the struct page count within that real mode environment as things like the sparsemem vmemmap mappings aren't accessible. This adds an API function realmode_pfn_to_page() to get page struct when MMU is off. This adds to MM a new function put_page_unless_one() which drops a page if counter is bigger than 1. It is going to be used when MMU is off (for example, real mode on PPC64) and we want to make sure that page release will not happen in real mode as it may crash the kernel in a horrible way. CONFIG_SPARSEMEM_VMEMMAP and CONFIG_FLATMEM are supported. Cc: linux-mm@kvack.org Cc: Benjamin Herrenschmidt Cc: Andrew Morton Reviewed-by: Paul Mackerras Signed-off-by: Paul Mackerras Signed-off-by: Alexey Kardashevskiy Signed-off-by: Benjamin Herrenschmidt --- arch/powerpc/include/asm/pgtable-ppc64.h | 2 ++ arch/powerpc/mm/init_64.c | 51 +++++++++++++++++++++++++++++++- include/linux/mm.h | 14 +++++++++ include/linux/page-flags.h | 4 ++- 4 files changed, 69 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/arch/powerpc/include/asm/pgtable-ppc64.h b/arch/powerpc/include/asm/pgtable-ppc64.h index 46db09414a1..4a191c47286 100644 --- a/arch/powerpc/include/asm/pgtable-ppc64.h +++ b/arch/powerpc/include/asm/pgtable-ppc64.h @@ -394,6 +394,8 @@ static inline void mark_hpte_slot_valid(unsigned char *hpte_slot_array, hpte_slot_array[index] = hidx << 4 | 0x1 << 3; } +struct page *realmode_pfn_to_page(unsigned long pfn); + static inline char *get_hpte_slot_array(pmd_t *pmdp) { /* diff --git a/arch/powerpc/mm/init_64.c b/arch/powerpc/mm/init_64.c index 8ed035d2edb..e3734edffa6 100644 --- a/arch/powerpc/mm/init_64.c +++ b/arch/powerpc/mm/init_64.c @@ -304,5 +304,54 @@ void register_page_bootmem_memmap(unsigned long section_nr, struct page *start_page, unsigned long size) { } -#endif /* CONFIG_SPARSEMEM_VMEMMAP */ +/* + * We do not have access to the sparsemem vmemmap, so we fallback to + * walking the list of sparsemem blocks which we already maintain for + * the sake of crashdump. In the long run, we might want to maintain + * a tree if performance of that linear walk becomes a problem. + * + * realmode_pfn_to_page functions can fail due to: + * 1) As real sparsemem blocks do not lay in RAM continously (they + * are in virtual address space which is not available in the real mode), + * the requested page struct can be split between blocks so get_page/put_page + * may fail. + * 2) When huge pages are used, the get_page/put_page API will fail + * in real mode as the linked addresses in the page struct are virtual + * too. + */ +struct page *realmode_pfn_to_page(unsigned long pfn) +{ + struct vmemmap_backing *vmem_back; + struct page *page; + unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift; + unsigned long pg_va = (unsigned long) pfn_to_page(pfn); + + for (vmem_back = vmemmap_list; vmem_back; vmem_back = vmem_back->list) { + if (pg_va < vmem_back->virt_addr) + continue; + + /* Check that page struct is not split between real pages */ + if ((pg_va + sizeof(struct page)) > + (vmem_back->virt_addr + page_size)) + return NULL; + + page = (struct page *) (vmem_back->phys + pg_va - + vmem_back->virt_addr); + return page; + } + + return NULL; +} +EXPORT_SYMBOL_GPL(realmode_pfn_to_page); + +#elif defined(CONFIG_FLATMEM) + +struct page *realmode_pfn_to_page(unsigned long pfn) +{ + struct page *page = pfn_to_page(pfn); + return page; +} +EXPORT_SYMBOL_GPL(realmode_pfn_to_page); + +#endif /* CONFIG_SPARSEMEM_VMEMMAP/CONFIG_FLATMEM */ diff --git a/include/linux/mm.h b/include/linux/mm.h index 8b6e55ee885..1a0668e5a4e 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -297,12 +297,26 @@ static inline int put_page_testzero(struct page *page) /* * Try to grab a ref unless the page has a refcount of zero, return false if * that is the case. + * This can be called when MMU is off so it must not access + * any of the virtual mappings. */ static inline int get_page_unless_zero(struct page *page) { return atomic_inc_not_zero(&page->_count); } +/* + * Try to drop a ref unless the page has a refcount of one, return false if + * that is the case. + * This is to make sure that the refcount won't become zero after this drop. + * This can be called when MMU is off so it must not access + * any of the virtual mappings. + */ +static inline int put_page_unless_one(struct page *page) +{ + return atomic_add_unless(&page->_count, -1, 1); +} + extern int page_is_ram(unsigned long pfn); /* Support for virtually mapped pages */ diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 6d53675c2b5..98ada58f994 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -329,7 +329,9 @@ static inline void set_page_writeback(struct page *page) * System with lots of page flags available. This allows separate * flags for PageHead() and PageTail() checks of compound pages so that bit * tests can be used in performance sensitive paths. PageCompound is - * generally not used in hot code paths. + * generally not used in hot code paths except arch/powerpc/mm/init_64.c + * and arch/powerpc/kvm/book3s_64_vio_hv.c which use it to detect huge pages + * and avoid handling those in real mode. */ __PAGEFLAG(Head, head) CLEARPAGEFLAG(Head, head) __PAGEFLAG(Tail, tail) -- cgit v1.2.3-70-g09d2 From a3e31b4588443f37d82195096c6b30dff1c152c2 Mon Sep 17 00:00:00 2001 From: Sudeep KarkadaNagesha Date: Wed, 18 Sep 2013 11:53:05 +0100 Subject: of: Move definition of of_find_next_cache_node into common code. Since the definition of_find_next_cache_node is architecture independent, the existing definition in powerpc can be moved to driver/of/base.c Cc: Benjamin Herrenschmidt Cc: Grant Likely Cc: Rob Herring Signed-off-by: Sudeep KarkadaNagesha Signed-off-by: Benjamin Herrenschmidt --- arch/powerpc/include/asm/prom.h | 3 --- arch/powerpc/kernel/prom.c | 31 ------------------------------- drivers/of/base.c | 31 +++++++++++++++++++++++++++++++ include/linux/of.h | 2 ++ 4 files changed, 33 insertions(+), 34 deletions(-) (limited to 'include') diff --git a/arch/powerpc/include/asm/prom.h b/arch/powerpc/include/asm/prom.h index 7d0c7f3a717..bf09e5a065b 100644 --- a/arch/powerpc/include/asm/prom.h +++ b/arch/powerpc/include/asm/prom.h @@ -44,9 +44,6 @@ void of_parse_dma_window(struct device_node *dn, const __be32 *dma_window, extern void kdump_move_device_tree(void); -/* cache lookup */ -struct device_node *of_find_next_cache_node(struct device_node *np); - #ifdef CONFIG_NUMA extern int of_node_to_nid(struct device_node *device); #else diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c index 09be2759c31..4432fd86a6d 100644 --- a/arch/powerpc/kernel/prom.c +++ b/arch/powerpc/kernel/prom.c @@ -760,37 +760,6 @@ void __init early_init_devtree(void *params) * *******/ -/** - * of_find_next_cache_node - Find a node's subsidiary cache - * @np: node of type "cpu" or "cache" - * - * Returns a node pointer with refcount incremented, use - * of_node_put() on it when done. Caller should hold a reference - * to np. - */ -struct device_node *of_find_next_cache_node(struct device_node *np) -{ - struct device_node *child; - const phandle *handle; - - handle = of_get_property(np, "l2-cache", NULL); - if (!handle) - handle = of_get_property(np, "next-level-cache", NULL); - - if (handle) - return of_find_node_by_phandle(be32_to_cpup(handle)); - - /* OF on pmac has nodes instead of properties named "l2-cache" - * beneath CPU nodes. - */ - if (!strcmp(np->type, "cpu")) - for_each_child_of_node(np, child) - if (!strcmp(child->type, "cache")) - return child; - - return NULL; -} - /** * of_get_ibm_chip_id - Returns the IBM "chip-id" of a device * @np: device node of the device diff --git a/drivers/of/base.c b/drivers/of/base.c index 865d3f66c86..b2cee3db5ce 100644 --- a/drivers/of/base.c +++ b/drivers/of/base.c @@ -1884,3 +1884,34 @@ int of_device_is_stdout_path(struct device_node *dn) return of_stdout == dn; } EXPORT_SYMBOL_GPL(of_device_is_stdout_path); + +/** + * of_find_next_cache_node - Find a node's subsidiary cache + * @np: node of type "cpu" or "cache" + * + * Returns a node pointer with refcount incremented, use + * of_node_put() on it when done. Caller should hold a reference + * to np. + */ +struct device_node *of_find_next_cache_node(const struct device_node *np) +{ + struct device_node *child; + const phandle *handle; + + handle = of_get_property(np, "l2-cache", NULL); + if (!handle) + handle = of_get_property(np, "next-level-cache", NULL); + + if (handle) + return of_find_node_by_phandle(be32_to_cpup(handle)); + + /* OF on pmac has nodes instead of properties named "l2-cache" + * beneath CPU nodes. + */ + if (!strcmp(np->type, "cpu")) + for_each_child_of_node(np, child) + if (!strcmp(child->type, "cache")) + return child; + + return NULL; +} diff --git a/include/linux/of.h b/include/linux/of.h index f95aee391e3..c08c07e249b 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -226,6 +226,8 @@ static inline int of_get_child_count(const struct device_node *np) return num; } +/* cache lookup */ +extern struct device_node *of_find_next_cache_node(const struct device_node *); extern struct device_node *of_find_node_with_property( struct device_node *from, const char *prop_name); #define for_each_node_with_property(dn, prop_name) \ -- cgit v1.2.3-70-g09d2